You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/05/25 00:15:04 UTC
(hudi) branch master updated: [HUDI-7776] Simplify HoodieStorage instance fetching (#11259)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new edb02d4c450 [HUDI-7776] Simplify HoodieStorage instance fetching (#11259)
edb02d4c450 is described below
commit edb02d4c450da7a7bdfb1ca7c13d20bf255ee335
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Fri May 24 17:14:57 2024 -0700
[HUDI-7776] Simplify HoodieStorage instance fetching (#11259)
---
.../hudi/cli/commands/ArchivedCommitsCommand.java | 2 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 23 +++--
.../apache/hudi/cli/commands/RepairsCommand.java | 2 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 2 +-
.../hudi/client/heartbeat/HeartbeatUtils.java | 5 +-
.../client/timeline/HoodieTimelineArchiver.java | 7 +-
.../hudi/client/timeline/LSMTimelineWriter.java | 4 +-
.../hudi/client/transaction/lock/LockManager.java | 3 +-
.../lock/metrics/HoodieLockMetrics.java | 14 +--
.../org/apache/hudi/index/HoodieIndexUtils.java | 7 +-
.../hudi/index/bloom/HoodieGlobalBloomIndex.java | 2 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 6 +-
.../hudi/index/simple/HoodieGlobalSimpleIndex.java | 2 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 2 +-
.../hudi/io/HoodieKeyLocationFetchHandle.java | 4 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieReadHandle.java | 8 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 8 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 6 +-
.../java/org/apache/hudi/table/HoodieTable.java | 10 +-
.../table/action/clean/CleanActionExecutor.java | 6 +-
.../hudi/table/action/clean/CleanPlanner.java | 4 +-
.../PartitionAwareClusteringPlanStrategy.java | 3 +-
.../action/commit/BaseCommitActionExecutor.java | 2 +-
.../table/action/commit/HoodieMergeHelper.java | 13 +--
.../compact/RunCompactionActionExecutor.java | 2 +-
.../BaseHoodieCompactionPlanGenerator.java | 3 +-
.../index/RecordBasedIndexingCatchupTask.java | 3 +-
.../table/action/index/RunIndexActionExecutor.java | 6 +-
.../action/index/ScheduleIndexActionExecutor.java | 2 +-
.../action/restore/BaseRestoreActionExecutor.java | 2 +-
.../rollback/BaseRollbackActionExecutor.java | 4 +-
.../rollback/ListingBasedRollbackStrategy.java | 2 +-
.../action/savepoint/SavepointActionExecutor.java | 3 +-
.../action/ttl/strategy/PartitionTTLStrategy.java | 3 +-
.../hudi/table/marker/DirectWriteMarkers.java | 2 +-
.../table/marker/MarkerBasedRollbackUtils.java | 2 +-
.../hudi/avro/TestHoodieAvroParquetWriter.java | 16 ++--
.../hudi/metrics/TestHoodieConsoleMetrics.java | 4 +-
.../hudi/metrics/TestHoodieGraphiteMetrics.java | 4 +-
.../apache/hudi/metrics/TestHoodieJmxMetrics.java | 4 +-
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 4 +-
.../datadog/TestDatadogMetricsReporter.java | 10 +-
.../org/apache/hudi/metrics/m3/TestM3Metrics.java | 21 ++---
.../metrics/prometheus/TestPrometheusReporter.java | 6 +-
.../prometheus/TestPushGateWayReporter.java | 6 +-
.../GenericRecordValidationTestUtils.java | 7 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 2 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 7 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 2 +-
.../FlinkDeletePartitionCommitActionExecutor.java | 2 +-
.../index/bloom/TestFlinkHoodieBloomIndex.java | 2 +-
.../run/strategy/JavaExecutionStrategy.java | 7 +-
.../JavaHoodieBackedTableMetadataWriter.java | 2 +-
...vaInsertOverwriteTableCommitActionExecutor.java | 2 +-
.../client/TestHoodieJavaWriteClientInsert.java | 8 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 10 +-
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 2 +-
.../commit/TestJavaCopyOnWriteActionExecutor.java | 14 +--
.../testutils/HoodieJavaClientTestHarness.java | 12 ++-
.../MultipleSparkJobExecutionStrategy.java | 14 ++-
.../strategy/SingleSparkJobExecutionStrategy.java | 3 +-
.../client/validator/SparkPreCommitValidator.java | 2 +-
.../index/bloom/HoodieFileProbingFunction.java | 3 +-
.../io/storage/HoodieSparkFileReaderFactory.java | 18 ++--
.../io/storage/HoodieSparkFileWriterFactory.java | 11 ++-
.../hudi/io/storage/HoodieSparkIOFactory.java | 15 +--
.../hudi/io/storage/HoodieSparkParquetReader.java | 32 +++----
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 4 +-
.../SparkHoodieBackedTableMetadataWriter.java | 2 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 2 +-
.../SparkDeletePartitionCommitActionExecutor.java | 2 +-
...rkInsertOverwriteTableCommitActionExecutor.java | 3 +-
.../SparkFileFormatInternalRowReaderContext.scala | 11 +--
.../org/apache/hudi/client/TestClientRollback.java | 4 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 2 +-
.../functional/TestExternalPathHandling.java | 3 +-
.../functional/TestHoodieBackedMetadata.java | 33 ++++---
.../functional/TestHoodieBackedTableMetadata.java | 11 ++-
.../TestHoodieClientOnCopyOnWriteStorage.java | 30 +++---
.../TestRemoteFileSystemViewWithMetadataTable.java | 2 +-
.../hudi/index/bloom/TestHoodieBloomIndex.java | 2 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 2 +-
.../storage/TestHoodieAvroFileWriterFactory.java | 8 +-
.../commit/TestCopyOnWriteActionExecutor.java | 6 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 3 +-
.../TestTimelineServerBasedWriteMarkers.java | 3 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 6 +-
.../hudi/testutils/FunctionalTestHarness.java | 3 +-
.../testutils/HoodieSparkClientTestHarness.java | 5 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 7 +-
.../hudi/common/config/HoodieStorageConfig.java | 12 ++-
.../hudi/common/engine/HoodieReaderContext.java | 4 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 12 ++-
.../hudi/common/model/HoodieCommitMetadata.java | 16 ++--
.../hudi/common/model/HoodiePartitionMetadata.java | 2 +-
.../hudi/common/table/HoodieTableMetaClient.java | 7 +-
.../hudi/common/table/TableSchemaResolver.java | 4 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 7 +-
.../table/log/block/HoodieAvroDataBlock.java | 4 +-
.../common/table/log/block/HoodieCommandBlock.java | 4 +-
.../common/table/log/block/HoodieCorruptBlock.java | 4 +-
.../common/table/log/block/HoodieDataBlock.java | 8 +-
.../common/table/log/block/HoodieDeleteBlock.java | 4 +-
.../table/log/block/HoodieHFileDataBlock.java | 28 +++---
.../common/table/log/block/HoodieLogBlock.java | 14 +--
.../table/log/block/HoodieParquetDataBlock.java | 17 ++--
.../common/table/read/HoodieFileGroupReader.java | 15 ++-
.../table/timeline/HoodieActiveTimeline.java | 8 +-
.../table/timeline/HoodieArchivedTimeline.java | 2 +-
.../common/table/view/FileSystemViewManager.java | 2 +-
.../apache/hudi/common/util/FileFormatUtils.java | 101 ++++++++++-----------
.../hudi/common/util/InternalSchemaCache.java | 16 ++--
.../io/FileBasedInternalSchemaStorageManager.java | 15 +--
.../hudi/io/storage/HoodieFileReaderFactory.java | 7 +-
.../hudi/io/storage/HoodieFileWriterFactory.java | 16 ++--
.../apache/hudi/io/storage/HoodieIOFactory.java | 13 ++-
.../io/storage/HoodieNativeAvroHFileReader.java | 13 +--
.../hudi/metadata/AbstractHoodieTableMetadata.java | 7 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 25 +++--
.../metadata/FileSystemBackedTableMetadata.java | 29 +++---
.../hudi/metadata/HoodieBackedTableMetadata.java | 17 +++-
.../metadata/HoodieMetadataFileSystemView.java | 5 +-
.../hudi/metadata/HoodieMetadataMetrics.java | 6 +-
.../hudi/metadata/HoodieMetadataPayload.java | 11 ---
.../apache/hudi/metadata/HoodieTableMetadata.java | 24 +++--
.../hudi/metadata/HoodieTableMetadataUtil.java | 23 ++---
.../main/java/org/apache/hudi/metrics/Metrics.java | 16 ++--
.../apache/hudi/storage/HoodieStorageUtils.java | 13 ++-
.../table/read/TestHoodieFileGroupReaderBase.java | 4 +-
.../hudi/common/testutils/HoodieTestUtils.java | 6 ++
.../reader/HoodieFileGroupReaderTestUtils.java | 14 +--
.../testutils/reader/HoodieFileSliceTestUtils.java | 6 +-
.../testutils/reader/HoodieTestReaderContext.java | 4 +-
.../HoodieBackedTestDelayedTableMetadata.java | 4 +-
.../hudi/sink/bootstrap/BootstrapOperator.java | 6 +-
.../hudi/sink/clustering/ClusteringOperator.java | 6 +-
.../sink/partitioner/profile/WriteProfiles.java | 5 +-
.../java/org/apache/hudi/source/FileIndex.java | 10 +-
.../hudi/source/stats/ColumnStatsIndices.java | 4 +-
.../hudi/table/format/InternalSchemaManager.java | 6 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 5 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 2 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 2 +-
.../common/config/DFSPropertiesConfiguration.java | 3 +-
.../common/table/log/HoodieLogFormatWriter.java | 2 +-
.../org/apache/hudi/common/util/HFileUtils.java | 29 +++---
.../java/org/apache/hudi/common/util/OrcUtils.java | 60 ++++++------
.../org/apache/hudi/common/util/ParquetUtils.java | 101 ++++++++++-----------
.../io/hadoop/HoodieAvroFileReaderFactory.java | 19 ++--
.../io/hadoop/HoodieAvroFileWriterFactory.java | 16 ++--
.../apache/hudi/io/hadoop/HoodieAvroOrcReader.java | 22 ++---
.../hudi/io/hadoop/HoodieAvroParquetReader.java | 19 ++--
.../{storage => hadoop}/HoodieHadoopIOFactory.java | 24 ++---
.../hudi/storage/hadoop/HoodieHadoopStorage.java | 30 ++++--
.../common/functional/TestHoodieLogFormat.java | 4 +-
.../hudi/common/table/TestTableSchemaResolver.java | 7 +-
.../table/log/block/TestHoodieDeleteBlock.java | 2 +-
.../reader/HoodieFileGroupReaderTestHarness.java | 3 +-
.../util/TestDFSPropertiesConfiguration.java | 6 +-
.../apache/hudi/common/util/TestParquetUtils.java | 14 +--
.../io/hadoop/TestHoodieAvroFileReaderFactory.java | 13 ++-
.../hadoop/TestHoodieHBaseHFileReaderWriter.java | 21 ++---
.../io/hadoop/TestHoodieHFileReaderWriter.java | 18 ++--
.../io/hadoop/TestHoodieHFileReaderWriterBase.java | 42 ++++-----
.../hudi/io/hadoop/TestHoodieOrcReaderWriter.java | 13 +--
.../hudi/io/hadoop/TestHoodieReaderWriterBase.java | 43 +++++----
.../TestFileSystemBackedTableMetadata.java | 13 +--
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 2 +-
.../hudi/hadoop/HoodieHFileRecordReader.java | 3 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 5 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 5 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 4 +-
.../reader/DFSHoodieDatasetInputReader.java | 4 +-
.../org/apache/hudi/storage/HoodieStorage.java | 25 ++++-
...lkInsertOverwriteTableCommitActionExecutor.java | 4 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 7 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 17 ++--
.../src/main/scala/org/apache/hudi/Iterators.scala | 2 +-
.../org/apache/hudi/SparkBaseIndexSupport.scala | 5 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 2 +-
...odieFileGroupReaderBasedParquetFileFormat.scala | 4 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 20 ++--
.../hudi/command/RepairHoodieTableCommand.scala | 16 +++-
.../sql/hudi/streaming/HoodieStreamSource.scala | 3 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 14 +--
.../RepairMigratePartitionMetaProcedure.scala | 3 +-
.../ShowColumnStatsOverlapProcedure.scala | 5 +-
.../procedures/ShowInvalidParquetProcedure.scala | 7 +-
.../ShowMetadataTableColumnStatsProcedure.scala | 5 +-
.../ShowMetadataTableFilesProcedure.scala | 4 +-
.../ShowMetadataTablePartitionsProcedure.scala | 7 +-
.../ShowMetadataTableStatsProcedure.scala | 4 +-
.../ValidateMetadataTableFilesProcedure.scala | 5 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 18 ++--
.../org/apache/hudi/functional/TestBootstrap.java | 14 +--
.../TestSparkConsistentBucketClustering.java | 4 +-
.../row/TestHoodieInternalRowParquetWriter.java | 4 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 1 +
.../apache/hudi/functional/TestCOWDataSource.scala | 8 +-
.../hudi/functional/TestColumnStatsIndex.scala | 8 +-
.../TestMetadataTableWithSparkDataSource.scala | 13 ++-
.../hudi/dml/TestPartialUpdateForMergeInto.scala | 3 +-
.../Spark3ParquetSchemaEvolutionUtils.scala | 10 +-
.../Spark30LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark31LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark32LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark33LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark34LegacyHoodieParquetFileFormat.scala | 5 +-
.../Spark35LegacyHoodieParquetFileFormat.scala | 4 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 1 +
.../hudi/sync/common/util/ManifestFileWriter.java | 2 +-
.../hudi/utilities/HoodieDataTableValidator.java | 2 +-
.../utilities/HoodieMetadataTableValidator.java | 16 ++--
.../apache/hudi/utilities/HoodieRepairTool.java | 2 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 4 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 8 +-
.../org/apache/hudi/utilities/TableSizeStats.java | 4 +-
.../deltastreamer/HoodieDeltaStreamerMetrics.java | 10 +-
.../ingestion/HoodieIngestionMetrics.java | 12 +--
.../hudi/utilities/perf/TimelineServerPerf.java | 5 +-
.../utilities/streamer/HoodieStreamerMetrics.java | 12 +--
.../apache/hudi/utilities/streamer/StreamSync.java | 7 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 4 +-
.../TestHoodieMetadataTableValidator.java | 4 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 5 +-
228 files changed, 1120 insertions(+), 1000 deletions(-)
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java
index 50e71f370db..9c4f70e2025 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java
@@ -110,7 +110,7 @@ public class ArchivedCommitsCommand {
StoragePath archivePath = folder != null && !folder.isEmpty()
? new StoragePath(metaClient.getMetaPath(), folder)
: new StoragePath(metaClient.getArchivePath(), ".commits_.archive*");
- HoodieStorage storage = HoodieStorageUtils.getStorage(metaClient.getBasePathV2(), HoodieCLI.conf);
+ HoodieStorage storage = metaClient.getStorage();
List<StoragePathInfo> pathInfoList = storage.globEntries(archivePath);
List<Comparable[]> allStats = new ArrayList<>();
for (StoragePathInfo pathInfo : pathInfoList) {
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
index 516901a25c4..cabd9fdcaf6 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
@@ -187,10 +187,10 @@ public class MetadataCommand {
@ShellMethod(key = "metadata stats", value = "Print stats about the metadata")
public String stats() throws IOException {
- HoodieCLI.getTableMetaClient();
+ HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
- try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(HoodieCLI.conf),
- config, HoodieCLI.basePath)) {
+ try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(
+ new HoodieLocalEngineContext(HoodieCLI.conf), metaClient.getStorage(), config, HoodieCLI.basePath)) {
Map<String, String> stats = metadata.stats();
final List<Comparable[]> rows = new ArrayList<>();
@@ -212,11 +212,11 @@ public class MetadataCommand {
public String listPartitions(
@ShellOption(value = "--sparkMaster", defaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master
) throws IOException {
- HoodieCLI.getTableMetaClient();
+ HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
initJavaSparkContext(Option.of(master));
HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
- try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), config,
- HoodieCLI.basePath)) {
+ try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(
+ new HoodieSparkEngineContext(jsc), metaClient.getStorage(), config, HoodieCLI.basePath)) {
if (!metadata.enabled()) {
return "[ERROR] Metadata Table not enabled/initialized\n\n";
@@ -241,10 +241,10 @@ public class MetadataCommand {
@ShellMethod(key = "metadata list-files", value = "Print a list of all files in a partition from the metadata")
public String listFiles(
@ShellOption(value = {"--partition"}, help = "Name of the partition to list files", defaultValue = "") final String partition) throws IOException {
- HoodieCLI.getTableMetaClient();
+ HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
try (HoodieBackedTableMetadata metaReader = new HoodieBackedTableMetadata(
- new HoodieLocalEngineContext(HoodieCLI.conf), config, HoodieCLI.basePath)) {
+ new HoodieLocalEngineContext(HoodieCLI.conf), metaClient.getStorage(), config, HoodieCLI.basePath)) {
if (!metaReader.enabled()) {
return "[ERROR] Metadata Table not enabled/initialized\n\n";
@@ -278,18 +278,17 @@ public class MetadataCommand {
public String validateFiles(
@ShellOption(value = {"--verbose"}, help = "Print all file details", defaultValue = "false") final boolean verbose)
throws IOException {
- HoodieCLI.getTableMetaClient();
+ HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
HoodieBackedTableMetadata metadataReader = new HoodieBackedTableMetadata(
- new HoodieLocalEngineContext(HoodieCLI.conf), config, HoodieCLI.basePath);
+ new HoodieLocalEngineContext(HoodieCLI.conf), metaClient.getStorage(), config, HoodieCLI.basePath);
if (!metadataReader.enabled()) {
return "[ERROR] Metadata Table not enabled/initialized\n\n";
}
FileSystemBackedTableMetadata fsMetaReader = new FileSystemBackedTableMetadata(new HoodieLocalEngineContext(HoodieCLI.conf),
- HoodieCLI.getTableMetaClient().getTableConfig(), HoodieCLI.conf,
- HoodieCLI.basePath);
+ metaClient.getTableConfig(), metaClient.getStorage(), HoodieCLI.basePath);
HoodieMetadataConfig fsConfig = HoodieMetadataConfig.newBuilder().enable(false).build();
HoodieTimer timer = HoodieTimer.start();
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
index 0f37190eee6..57ec8ccf57b 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
@@ -227,7 +227,7 @@ public class RepairsCommand {
HoodieLocalEngineContext engineContext = new HoodieLocalEngineContext(HoodieCLI.conf);
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
List<String> partitionPaths =
- FSUtils.getAllPartitionPaths(engineContext, client.getBasePath(), false);
+ FSUtils.getAllPartitionPaths(engineContext, client.getStorage(), client.getBasePath(), false);
StoragePath basePath = client.getBasePathV2();
String[][] rows = new String[partitionPaths.size()][];
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
index f982a0e4e22..4dd09a4af62 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
@@ -102,7 +102,7 @@ public abstract class BaseHoodieClient implements Serializable, AutoCloseable {
this.heartbeatClient = new HoodieHeartbeatClient(storage, this.basePath,
clientConfig.getHoodieClientHeartbeatIntervalInMs(),
clientConfig.getHoodieClientHeartbeatTolerableMisses());
- this.metrics = new HoodieMetrics(config, context.getStorageConf());
+ this.metrics = new HoodieMetrics(config, storage);
this.txnManager = new TransactionManager(config, storage);
this.timeGenerator = TimeGenerators.getTimeGenerator(
config.getTimeGeneratorConfig(), HadoopFSUtils.getStorageConf(hadoopConf));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java
index dcdc45932c2..197ef2e6119 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java
@@ -22,8 +22,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.slf4j.Logger;
@@ -100,8 +100,7 @@ public class HeartbeatUtils {
if (config.getFailedWritesCleanPolicy().isLazy() && heartbeatClient.isHeartbeatExpired(instantTime)) {
throw new HoodieException(
"Heartbeat for instant " + instantTime + " has expired, last heartbeat "
- + getLastHeartbeatTime(
- table.getMetaClient().getStorage(), config.getBasePath(), instantTime));
+ + getLastHeartbeatTime(table.getStorage(), config.getBasePath(), instantTime));
}
} catch (IOException io) {
throw new HoodieException("Unable to read heartbeat", io);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java
index f4ab6c76e13..a50eddf8bb7 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java
@@ -81,12 +81,12 @@ public class HoodieTimelineArchiver<T extends HoodieAvroPayload, I, K, O> {
this.config = config;
this.table = table;
this.metaClient = table.getMetaClient();
- this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
+ this.txnManager = new TransactionManager(config, table.getStorage());
this.timelineWriter = LSMTimelineWriter.getInstance(config, table);
Pair<Integer, Integer> minAndMaxInstants = getMinAndMaxInstantsToKeep(table, metaClient);
this.minInstantsToKeep = minAndMaxInstants.getLeft();
this.maxInstantsToKeep = minAndMaxInstants.getRight();
- this.metrics = new HoodieMetrics(config, table.getStorageConf());
+ this.metrics = new HoodieMetrics(config, metaClient.getStorage());
}
public int archiveIfRequired(HoodieEngineContext context) throws IOException {
@@ -207,7 +207,8 @@ public class HoodieTimelineArchiver<T extends HoodieAvroPayload, I, K, O> {
// 4. If metadata table is enabled, do not archive instants which are more recent than the last compaction on the
// metadata table.
if (table.getMetaClient().getTableConfig().isMetadataTableAvailable()) {
- try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(table.getContext(), config.getMetadataConfig(), config.getBasePath())) {
+ try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(
+ table.getContext(), table.getStorage(), config.getMetadataConfig(), config.getBasePath())) {
Option<String> latestCompactionTime = tableMetadata.getLatestCompactionTime();
if (!latestCompactionTime.isPresent()) {
LOG.info("Not archiving as there is no compaction yet on the metadata table");
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LSMTimelineWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LSMTimelineWriter.java
index f910d44f734..4986c22fd05 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LSMTimelineWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LSMTimelineWriter.java
@@ -269,7 +269,7 @@ public class LSMTimelineWriter {
try (HoodieFileWriter writer = openWriter(new StoragePath(metaClient.getArchivePath(), compactedFileName))) {
for (String fileName : candidateFiles) {
// Read the input source file
- try (HoodieAvroParquetReader reader = (HoodieAvroParquetReader) HoodieIOFactory.getIOFactory(metaClient.getStorageConf())
+ try (HoodieAvroParquetReader reader = (HoodieAvroParquetReader) HoodieIOFactory.getIOFactory(metaClient.getStorage())
.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(config, new StoragePath(metaClient.getArchivePath(), fileName))) {
// Read the meta entry
@@ -382,7 +382,7 @@ public class LSMTimelineWriter {
private HoodieFileWriter openWriter(StoragePath filePath) {
try {
- return HoodieFileWriterFactory.getFileWriter("", filePath, metaClient.getStorageConf(), getOrCreateWriterConfig(),
+ return HoodieFileWriterFactory.getFileWriter("", filePath, metaClient.getStorage(), getOrCreateWriterConfig(),
HoodieLSMTimelineInstant.getClassSchema(), taskContextSupplier, HoodieRecord.HoodieRecordType.AVRO);
} catch (IOException e) {
throw new HoodieException("Unable to initialize archiving writer", e);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
index f2920a7da50..39df3c3fe43 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
@@ -30,6 +30,7 @@ import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieLockException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
@@ -69,7 +70,7 @@ public class LockManager implements Serializable, AutoCloseable {
Integer.parseInt(HoodieLockConfig.LOCK_ACQUIRE_CLIENT_NUM_RETRIES.defaultValue()));
maxWaitTimeInMs = lockConfiguration.getConfig().getLong(LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY,
Long.parseLong(HoodieLockConfig.LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS.defaultValue()));
- metrics = new HoodieLockMetrics(writeConfig, storageConf);
+ metrics = new HoodieLockMetrics(writeConfig, new HoodieHadoopStorage(fs));
lockRetryHelper = new RetryHelper<>(maxWaitTimeInMs, maxRetries, maxWaitTimeInMs,
Arrays.asList(HoodieLockException.class, InterruptedException.class), "acquire lock");
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/metrics/HoodieLockMetrics.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/metrics/HoodieLockMetrics.java
index 7a793de5392..a67b6abbc98 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/metrics/HoodieLockMetrics.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/metrics/HoodieLockMetrics.java
@@ -18,16 +18,16 @@
package org.apache.hudi.client.transaction.lock.metrics;
+import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.metrics.Metrics;
+import org.apache.hudi.storage.HoodieStorage;
+
import com.codahale.metrics.Counter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SlidingWindowReservoir;
import com.codahale.metrics.Timer;
-import org.apache.hudi.common.util.HoodieTimer;
-import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.metrics.Metrics;
-import org.apache.hudi.storage.StorageConfiguration;
-
import java.util.concurrent.TimeUnit;
public class HoodieLockMetrics {
@@ -50,12 +50,12 @@ public class HoodieLockMetrics {
private static final Object REGISTRY_LOCK = new Object();
private Metrics metrics;
- public HoodieLockMetrics(HoodieWriteConfig writeConfig, StorageConfiguration<?> storageConf) {
+ public HoodieLockMetrics(HoodieWriteConfig writeConfig, HoodieStorage storage) {
this.isMetricsEnabled = writeConfig.isLockingMetricsEnabled();
this.writeConfig = writeConfig;
if (isMetricsEnabled) {
- metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storageConf);
+ metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storage);
MetricRegistry registry = metrics.getRegistry();
lockAttempts = registry.counter(getMetricsName(LOCK_ACQUIRE_ATTEMPTS_COUNTER_NAME));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
index 860185ed176..da713ac552b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
@@ -47,7 +47,7 @@ import org.apache.hudi.io.HoodieMergedReadHandle;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.metadata.MetadataPartitionType;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
@@ -176,14 +176,15 @@ public class HoodieIndexUtils {
*
* @param filePath - File to filter keys from
* @param candidateRecordKeys - Candidate keys to filter
+ * @param storage
* @return List of pairs of candidate keys and positions that are available in the file
*/
public static List<Pair<String, Long>> filterKeysFromFile(StoragePath filePath,
List<String> candidateRecordKeys,
- StorageConfiguration<?> configuration) throws HoodieIndexException {
+ HoodieStorage storage) throws HoodieIndexException {
checkArgument(FSUtils.isBaseFile(filePath));
List<Pair<String, Long>> foundRecordKeys = new ArrayList<>();
- try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(configuration)
+ try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(storage)
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, filePath)) {
// Load all rowKeys from the file, to double-confirm
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java
index 178bc6156e7..25ca3984bde 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java
@@ -57,7 +57,7 @@ public class HoodieGlobalBloomIndex extends HoodieBloomIndex {
List<Pair<String, BloomIndexFileInfo>> loadColumnRangesFromFiles(List<String> partitions, final HoodieEngineContext context,
final HoodieTable hoodieTable) {
HoodieTableMetaClient metaClient = hoodieTable.getMetaClient();
- List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), metaClient.getBasePath());
+ List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(context, metaClient.getStorage(), config.getMetadataConfig(), metaClient.getBasePath());
return super.loadColumnRangesFromFiles(allPartitionPaths, context, hoodieTable);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
index 6023b17ce0d..00ee16b285d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
@@ -187,7 +187,7 @@ public class ConsistentBucketIndexUtils {
* @return true if the metadata is saved successfully
*/
public static boolean saveMetadata(HoodieTable table, HoodieConsistentHashingMetadata metadata, boolean overwrite) {
- HoodieStorage storage = table.getMetaClient().getStorage();
+ HoodieStorage storage = table.getStorage();
StoragePath dir = FSUtils.constructAbsolutePath(
table.getMetaClient().getHashingMetadataPath(), metadata.getPartitionPath());
StoragePath fullPath = new StoragePath(dir, metadata.getFilename());
@@ -211,7 +211,7 @@ public class ConsistentBucketIndexUtils {
* @throws IOException
*/
private static void createCommitMarker(HoodieTable table, Path fileStatus, Path partitionPath) throws IOException {
- HoodieStorage storage = table.getMetaClient().getStorage();
+ HoodieStorage storage = table.getStorage();
StoragePath fullPath = new StoragePath(convertToStoragePath(partitionPath),
getTimestampFromFile(fileStatus.getName()) + HASHING_METADATA_COMMIT_FILE_SUFFIX);
if (storage.exists(fullPath)) {
@@ -240,7 +240,7 @@ public class ConsistentBucketIndexUtils {
if (metaFile == null) {
return Option.empty();
}
- try (InputStream is = table.getMetaClient().getStorage().open(convertToStoragePath(metaFile.getPath()))) {
+ try (InputStream is = table.getStorage().open(convertToStoragePath(metaFile.getPath()))) {
byte[] content = FileIOUtils.readAsByteArray(is);
return Option.of(HoodieConsistentHashingMetadata.fromBytes(content));
} catch (FileNotFoundException e) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/simple/HoodieGlobalSimpleIndex.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/simple/HoodieGlobalSimpleIndex.java
index 5256b036fde..7432d606839 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/simple/HoodieGlobalSimpleIndex.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/simple/HoodieGlobalSimpleIndex.java
@@ -94,7 +94,7 @@ public class HoodieGlobalSimpleIndex extends HoodieSimpleIndex {
private List<Pair<String, HoodieBaseFile>> getAllBaseFilesInTable(
final HoodieEngineContext context, final HoodieTable hoodieTable) {
HoodieTableMetaClient metaClient = hoodieTable.getMetaClient();
- List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), metaClient.getBasePath());
+ List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(context, metaClient.getStorage(), config.getMetadataConfig(), metaClient.getBasePath());
// Obtain the latest data files from all the partitions.
return getLatestBaseFilesForAllPartitions(allPartitionPaths, context, hoodieTable);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
index 07f30c1e3fa..dc61749bf93 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
@@ -105,7 +105,7 @@ public class HoodieCreateHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
createMarkerFile(partitionPath,
FSUtils.makeBaseFileName(this.instantTime, this.writeToken, this.fileId, hoodieTable.getBaseFileExtension()));
this.fileWriter =
- HoodieFileWriterFactory.getFileWriter(instantTime, path, hoodieTable.getStorageConf(), config,
+ HoodieFileWriterFactory.getFileWriter(instantTime, path, hoodieTable.getStorage(), config,
writeSchemaWithMetaFields, this.taskContextSupplier, config.getRecordMerger().getRecordType());
} catch (IOException e) {
throw new HoodieInsertException("Failed to initialize HoodieStorageWriter for path " + path, e);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
index 734a012d4bb..835f102f414 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
@@ -52,9 +52,9 @@ public class HoodieKeyLocationFetchHandle<T, I, K, O> extends HoodieReadHandle<T
private List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieBaseFile baseFile) {
FileFormatUtils fileFormatUtils = FileFormatUtils.getInstance(baseFile.getStoragePath());
if (keyGeneratorOpt.isPresent()) {
- return fileFormatUtils.fetchRecordKeysWithPositions(hoodieTable.getStorageConf(), baseFile.getStoragePath(), keyGeneratorOpt);
+ return fileFormatUtils.fetchRecordKeysWithPositions(hoodieTable.getStorage(), baseFile.getStoragePath(), keyGeneratorOpt);
} else {
- return fileFormatUtils.fetchRecordKeysWithPositions(hoodieTable.getStorageConf(), baseFile.getStoragePath());
+ return fileFormatUtils.fetchRecordKeysWithPositions(hoodieTable.getStorage(), baseFile.getStoragePath());
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java
index 77c4a841f3c..ca0c009ffda 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java
@@ -101,7 +101,7 @@ public class HoodieKeyLookupHandle<T, I, K, O> extends HoodieReadHandle<T, I, K,
HoodieBaseFile baseFile = getLatestBaseFile();
List<Pair<String, Long>> matchingKeysAndPositions = HoodieIndexUtils.filterKeysFromFile(
- baseFile.getStoragePath(), candidateRecordKeys, hoodieTable.getStorageConf());
+ baseFile.getStoragePath(), candidateRecordKeys, hoodieTable.getStorage());
LOG.info(
String.format("Total records (%d), bloom filter candidates (%d)/fp(%d), actual matches (%d)", totalKeysChecked,
candidateRecordKeys.size(), candidateRecordKeys.size() - matchingKeysAndPositions.size(), matchingKeysAndPositions.size()));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
index f12b536a1e0..b5dd3374d27 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
@@ -198,7 +198,7 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
createMarkerFile(partitionPath, newFilePath.getName());
// Create the writer for writing the new version file
- fileWriter = HoodieFileWriterFactory.getFileWriter(instantTime, newFilePath, hoodieTable.getStorageConf(),
+ fileWriter = HoodieFileWriterFactory.getFileWriter(instantTime, newFilePath, hoodieTable.getStorage(),
config, writeSchemaWithMetaFields, taskContextSupplier, recordMerger.getRecordType());
} catch (IOException io) {
LOG.error("Error in update task at commit " + instantTime, io);
@@ -470,7 +470,7 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
}
long oldNumWrites = 0;
- try (HoodieFileReader reader = HoodieIOFactory.getIOFactory(hoodieTable.getStorageConf())
+ try (HoodieFileReader reader = HoodieIOFactory.getIOFactory(hoodieTable.getStorage())
.getReaderFactory(this.recordMerger.getRecordType())
.getFileReader(config, oldFilePath)) {
oldNumWrites = reader.getTotalRecords();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java
index 71d691ad580..66cf5b3e18d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java
@@ -52,7 +52,7 @@ public abstract class HoodieReadHandle<T, I, K, O> extends HoodieIOHandle<T, I,
@Override
public HoodieStorage getStorage() {
- return hoodieTable.getMetaClient().getStorage();
+ return hoodieTable.getStorage();
}
public Pair<String, String> getPartitionPathFileIDPair() {
@@ -69,12 +69,14 @@ public abstract class HoodieReadHandle<T, I, K, O> extends HoodieIOHandle<T, I,
}
protected HoodieFileReader createNewFileReader() throws IOException {
- return HoodieIOFactory.getIOFactory(hoodieTable.getStorageConf()).getReaderFactory(this.config.getRecordMerger().getRecordType())
+ return HoodieIOFactory.getIOFactory(hoodieTable.getStorage())
+ .getReaderFactory(this.config.getRecordMerger().getRecordType())
.getFileReader(config, getLatestBaseFile().getStoragePath());
}
protected HoodieFileReader createNewFileReader(HoodieBaseFile hoodieBaseFile) throws IOException {
- return HoodieIOFactory.getIOFactory(hoodieTable.getStorageConf()).getReaderFactory(this.config.getRecordMerger().getRecordType())
+ return HoodieIOFactory.getIOFactory(hoodieTable.getStorage())
+ .getReaderFactory(this.config.getRecordMerger().getRecordType())
.getFileReader(config, hoodieBaseFile.getStoragePath());
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
index b9b25b870a2..a251a5f7614 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
@@ -203,7 +203,7 @@ public abstract class HoodieWriteHandle<T, I, K, O> extends HoodieIOHandle<T, I,
@Override
public HoodieStorage getStorage() {
- return hoodieTable.getMetaClient().getStorage();
+ return hoodieTable.getStorage();
}
public HoodieWriteConfig getConfig() {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index b3791bd7c99..d4b2a7333f5 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -68,12 +68,12 @@ import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hudi.index.HoodieIndexUtils;
+import org.apache.hudi.io.HoodieMergedReadHandle;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
-import org.apache.hudi.io.HoodieMergedReadHandle;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.hudi.table.HoodieTable;
@@ -193,7 +193,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
}
try {
- this.metadata = new HoodieBackedTableMetadata(engineContext, dataWriteConfig.getMetadataConfig(), dataWriteConfig.getBasePath(), true);
+ this.metadata = new HoodieBackedTableMetadata(engineContext, dataMetaClient.getStorage(), dataWriteConfig.getMetadataConfig(), dataWriteConfig.getBasePath(), true);
this.metadataMetaClient = metadata.getMetadataMetaClient();
} catch (Exception e) {
throw new HoodieException("Could not open MDT for reads", e);
@@ -723,7 +723,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
// List all directories in parallel
engineContext.setJobStatus(this.getClass().getSimpleName(), "Listing " + numDirsToList + " partitions from filesystem");
List<DirectoryInfo> processedDirectories = engineContext.map(pathsToList.subList(0, numDirsToList), path -> {
- HoodieStorage storage = HoodieStorageUtils.getStorage(path, storageConf);
+ HoodieStorage storage = new HoodieHadoopStorage(path, storageConf);
String relativeDirPath = FSUtils.getRelativePartitionPath(storageBasePath, path);
return new DirectoryInfo(relativeDirPath, storage.listDirectEntries(path), initializationTime, pendingDataInstants);
}, numDirsToList);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
index d5020014036..49c02afa326 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
@@ -25,7 +25,7 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
@@ -110,11 +110,11 @@ public class HoodieMetrics {
private Counter compactionRequestedCounter = null;
private Counter compactionCompletedCounter = null;
- public HoodieMetrics(HoodieWriteConfig config, StorageConfiguration<?> storageConf) {
+ public HoodieMetrics(HoodieWriteConfig config, HoodieStorage storage) {
this.config = config;
this.tableName = config.getTableName();
if (config.isMetricsOn()) {
- metrics = Metrics.getInstance(config.getMetricsConfig(), storageConf);
+ metrics = Metrics.getInstance(config.getMetricsConfig(), storage);
this.rollbackTimerName = getMetricsName(TIMER_ACTION, HoodieTimeline.ROLLBACK_ACTION);
this.cleanTimerName = getMetricsName(TIMER_ACTION, HoodieTimeline.CLEAN_ACTION);
this.archiveTimerName = getMetricsName(TIMER_ACTION, ARCHIVE_ACTION);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
index adbd4112438..159828130bd 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -36,6 +36,8 @@ import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.engine.TaskContextSupplier;
+import org.apache.hudi.common.fs.ConsistencyGuard;
+import org.apache.hudi.common.fs.ConsistencyGuard.FileVisibility;
import org.apache.hudi.common.fs.ConsistencyGuardConfig;
import org.apache.hudi.common.fs.FailSafeConsistencyGuard;
import org.apache.hudi.common.fs.OptimisticConsistencyGuard;
@@ -67,8 +69,6 @@ import org.apache.hudi.exception.HoodieInsertException;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.exception.SchemaCompatibilityException;
-import org.apache.hudi.common.fs.ConsistencyGuard;
-import org.apache.hudi.common.fs.ConsistencyGuard.FileVisibility;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.metadata.HoodieTableMetadata;
@@ -146,7 +146,7 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().fromProperties(config.getMetadataConfig().getProps())
.build();
- this.metadata = HoodieTableMetadata.create(context, metadataConfig, config.getBasePath());
+ this.metadata = HoodieTableMetadata.create(context, metaClient.getStorage(), metadataConfig, config.getBasePath());
this.viewManager = getViewManager();
this.metaClient = metaClient;
@@ -318,6 +318,10 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
return metaClient.getStorageConf();
}
+ public HoodieStorage getStorage() {
+ return metaClient.getStorage();
+ }
+
/**
* Get the view of the file system for this table.
*/
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
index 2d7cf46793d..f8933457450 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
@@ -69,7 +69,7 @@ public class CleanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K,
public CleanActionExecutor(HoodieEngineContext context, HoodieWriteConfig config, HoodieTable<T, I, K, O> table, String instantTime, boolean skipLocking) {
super(context, config, table, instantTime);
- this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
+ this.txnManager = new TransactionManager(config, table.getStorage());
this.skipLocking = skipLocking;
}
@@ -91,7 +91,7 @@ public class CleanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K,
private static Stream<Pair<String, PartitionCleanStat>> deleteFilesFunc(Iterator<Pair<String, CleanFileInfo>> cleanFileInfo, HoodieTable table) {
Map<String, PartitionCleanStat> partitionCleanStatMap = new HashMap<>();
- FileSystem fs = (FileSystem) table.getMetaClient().getStorage().getFileSystem();
+ FileSystem fs = (FileSystem) table.getStorage().getFileSystem();
cleanFileInfo.forEachRemaining(partitionDelFileTuple -> {
String partitionPath = partitionDelFileTuple.getLeft();
@@ -152,7 +152,7 @@ public class CleanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K,
partitionsToBeDeleted.forEach(entry -> {
try {
if (!isNullOrEmpty(entry)) {
- deleteFileAndGetResult((FileSystem) table.getMetaClient().getStorage().getFileSystem(),
+ deleteFileAndGetResult((FileSystem) table.getStorage().getFileSystem(),
table.getMetaClient().getBasePath() + "/" + entry);
}
} catch (IOException e) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
index 651ea401087..3033e46516f 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
@@ -280,7 +280,7 @@ public class CleanPlanner<T, I, K, O> implements Serializable {
*/
private List<String> getPartitionPathsForFullCleaning() {
// Go to brute force mode of scanning all partitions
- return FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), config.getBasePath());
+ return FSUtils.getAllPartitionPaths(context, hoodieTable.getStorage(), config.getMetadataConfig(), config.getBasePath());
}
/**
@@ -468,7 +468,7 @@ public class CleanPlanner<T, I, K, O> implements Serializable {
HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(hoodieTable.getMetaClient(), hoodieTable.getActiveTimeline());
StoragePath fullPartitionPath = new StoragePath(hoodieTable.getMetaClient().getBasePathV2(), partitionPath);
fsView.addFilesToView(partitionPath, FSUtils.getAllDataFilesInPartition(
- hoodieTable.getMetaClient().getStorage(), fullPartitionPath));
+ hoodieTable.getStorage(), fullPartitionPath));
// use #getAllFileGroups(partitionPath) instead of #getAllFileGroups() to exclude the replaced file groups.
return fsView.getAllFileGroups(partitionPath).findAny().isPresent();
} catch (Exception ex) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
index c22c8b24eee..59789ab3f07 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
@@ -135,7 +135,8 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
if (StringUtils.isNullOrEmpty(partitionSelected)) {
// get matched partitions if set
- partitionPaths = getRegexPatternMatchedPartitions(config, FSUtils.getAllPartitionPaths(getEngineContext(), config.getMetadataConfig(), metaClient.getBasePath()));
+ partitionPaths = getRegexPatternMatchedPartitions(config, FSUtils.getAllPartitionPaths(
+ getEngineContext(), metaClient.getStorage(), config.getMetadataConfig(), metaClient.getBasePath()));
// filter the partition paths if needed to reduce list status
} else {
partitionPaths = Arrays.asList(partitionSelected.split(","));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
index bae32945917..47ad5f07d83 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
@@ -94,7 +94,7 @@ public abstract class BaseCommitActionExecutor<T, I, K, O, R>
this.taskContextSupplier = context.getTaskContextSupplier();
// TODO : Remove this once we refactor and move out autoCommit method from here, since the TxnManager is held in {@link BaseHoodieWriteClient}.
this.txnManagerOption = config.shouldAutoCommit()
- ? Option.of(new TransactionManager(config, table.getMetaClient().getStorage())) : Option.empty();
+ ? Option.of(new TransactionManager(config, table.getStorage())) : Option.empty();
if (this.txnManagerOption.isPresent() && this.txnManagerOption.get().isLockRequired()) {
// these txn metadata are only needed for auto commit when optimistic concurrent control is also enabled
this.lastCompletedTxn = TransactionUtils.getLastCompletedTxnInstantAndMetadata(table.getMetaClient());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java
index cfd9ff606dd..3f661e3b10b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java
@@ -38,7 +38,7 @@ import org.apache.hudi.internal.schema.utils.SerDeHelper;
import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.util.ExecutorFactory;
@@ -78,9 +78,9 @@ public class HoodieMergeHelper<T> extends BaseMergeHelper {
HoodieWriteConfig writeConfig = table.getConfig();
HoodieBaseFile baseFile = mergeHandle.baseFileForMerge();
- StorageConfiguration<?> storageConf = table.getStorageConf().newInstance();
HoodieRecord.HoodieRecordType recordType = table.getConfig().getRecordMerger().getRecordType();
- HoodieFileReader baseFileReader = HoodieIOFactory.getIOFactory(storageConf)
+ HoodieFileReader baseFileReader = HoodieIOFactory.getIOFactory(
+ table.getStorage().newInstance(mergeHandle.getOldFilePath(), table.getStorageConf().newInstance()))
.getReaderFactory(recordType)
.getFileReader(writeConfig, mergeHandle.getOldFilePath());
HoodieFileReader bootstrapFileReader = null;
@@ -111,10 +111,11 @@ public class HoodieMergeHelper<T> extends BaseMergeHelper {
Schema recordSchema;
if (baseFile.getBootstrapBaseFile().isPresent()) {
StoragePath bootstrapFilePath = baseFile.getBootstrapBaseFile().get().getStoragePath();
- StorageConfiguration<?> bootstrapFileConfig = table.getStorageConf().newInstance();
- bootstrapFileReader = HoodieIOFactory.getIOFactory(bootstrapFileConfig).getReaderFactory(recordType).newBootstrapFileReader(
+ HoodieStorage storage = table.getStorage().newInstance(
+ bootstrapFilePath, table.getStorageConf().newInstance());
+ bootstrapFileReader = HoodieIOFactory.getIOFactory(storage).getReaderFactory(recordType).newBootstrapFileReader(
baseFileReader,
- HoodieIOFactory.getIOFactory(bootstrapFileConfig).getReaderFactory(recordType)
+ HoodieIOFactory.getIOFactory(storage).getReaderFactory(recordType)
.getFileReader(writeConfig, bootstrapFilePath),
mergeHandle.getPartitionFields(),
mergeHandle.getPartitionValues());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/RunCompactionActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/RunCompactionActionExecutor.java
index 55e8ce7d23f..f8ce1fcdc2a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/RunCompactionActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/RunCompactionActionExecutor.java
@@ -73,7 +73,7 @@ public class RunCompactionActionExecutor<T> extends
this.operationType = operationType;
checkArgument(operationType == WriteOperationType.COMPACT || operationType == WriteOperationType.LOG_COMPACT,
"Only COMPACT and LOG_COMPACT is supported");
- metrics = new HoodieMetrics(config, context.getStorageConf());
+ metrics = new HoodieMetrics(config, table.getStorage());
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
index 9f6fb0d786a..9b1e91cca1f 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
@@ -82,7 +82,8 @@ public abstract class BaseHoodieCompactionPlanGenerator<T extends HoodieRecordPa
// TODO - rollback any compactions in flight
HoodieTableMetaClient metaClient = hoodieTable.getMetaClient();
CompletionTimeQueryView completionTimeQueryView = new CompletionTimeQueryView(metaClient);
- List<String> partitionPaths = FSUtils.getAllPartitionPaths(engineContext, writeConfig.getMetadataConfig(), metaClient.getBasePath());
+ List<String> partitionPaths = FSUtils.getAllPartitionPaths(
+ engineContext, metaClient.getStorage(), writeConfig.getMetadataConfig(), metaClient.getBasePath());
// filter the partition paths if needed to reduce list status
partitionPaths = filterPartitionPathsByStrategy(writeConfig, partitionPaths);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RecordBasedIndexingCatchupTask.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RecordBasedIndexingCatchupTask.java
index 53f357c03f7..04e51f9774e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RecordBasedIndexingCatchupTask.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RecordBasedIndexingCatchupTask.java
@@ -66,7 +66,8 @@ public class RecordBasedIndexingCatchupTask extends AbstractIndexingCatchupTask
private HoodieData<HoodieRecord> readRecordKeysFromFileSlices(HoodieInstant instant) throws IOException {
HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).build();
- HoodieTableMetadata metadata = HoodieTableMetadata.create(engineContext, metadataConfig, metaClient.getBasePathV2().toString(), false);
+ HoodieTableMetadata metadata = HoodieTableMetadata.create(
+ engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePathV2().toString(), false);
HoodieMetadataFileSystemView fsView = new HoodieMetadataFileSystemView(metaClient, metaClient.getActiveTimeline().filter(i -> i.equals(instant)), metadata);
// Collect the list of latest file slices present in each partition
List<String> partitions = metadata.getAllPartitionPaths();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
index 61e234534bc..ad9c072061d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
@@ -98,9 +98,9 @@ public class RunIndexActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I,
public RunIndexActionExecutor(HoodieEngineContext context, HoodieWriteConfig config, HoodieTable<T, I, K, O> table, String instantTime) {
super(context, config, table, instantTime);
- this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
+ this.txnManager = new TransactionManager(config, table.getStorage());
if (config.getMetadataConfig().isMetricsEnabled()) {
- this.metrics = Option.of(new HoodieMetadataMetrics(config.getMetricsConfig(), context.getStorageConf()));
+ this.metrics = Option.of(new HoodieMetadataMetrics(config.getMetricsConfig(), table.getStorage()));
} else {
this.metrics = Option.empty();
}
@@ -215,7 +215,7 @@ public class RunIndexActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I,
});
table.getMetaClient().getTableConfig().setValue(TABLE_METADATA_PARTITIONS_INFLIGHT.key(), String.join(",", inflightPartitions));
table.getMetaClient().getTableConfig().setValue(TABLE_METADATA_PARTITIONS.key(), String.join(",", completedPartitions));
- HoodieTableConfig.update(table.getMetaClient().getStorage(),
+ HoodieTableConfig.update(table.getStorage(),
table.getMetaClient().getMetaPath(), table.getMetaClient().getTableConfig().getProps());
// delete metadata partition
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java
index 7c1551bb0e1..87b9a929b2e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java
@@ -81,7 +81,7 @@ public class ScheduleIndexActionExecutor<T, I, K, O> extends BaseActionExecutor<
super(context, config, table, instantTime);
this.partitionIndexTypes = partitionIndexTypes;
this.partitionPaths = partitionPaths;
- this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
+ this.txnManager = new TransactionManager(config, table.getStorage());
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
index ad00fe052df..30a02f2b458 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
@@ -62,7 +62,7 @@ public abstract class BaseRestoreActionExecutor<T, I, K, O> extends BaseActionEx
String savepointToRestoreTimestamp) {
super(context, config, table, instantTime);
this.savepointToRestoreTimestamp = savepointToRestoreTimestamp;
- this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
+ this.txnManager = new TransactionManager(config, table.getStorage());
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
index 8dd7afe2f84..1ab943fea88 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
@@ -89,7 +89,7 @@ public abstract class BaseRollbackActionExecutor<T, I, K, O> extends BaseActionE
this.deleteInstants = deleteInstants;
this.skipTimelinePublish = skipTimelinePublish;
this.skipLocking = skipLocking;
- this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
+ this.txnManager = new TransactionManager(config, table.getStorage());
}
/**
@@ -174,7 +174,7 @@ public abstract class BaseRollbackActionExecutor<T, I, K, O> extends BaseActionE
&& !commitTimeline.findInstantsAfter(instantTimeToRollback, Integer.MAX_VALUE).empty()) {
// check if remnants are from a previous LAZY rollback config, if yes, let out of order rollback continue
try {
- if (!HoodieHeartbeatClient.heartbeatExists(table.getMetaClient().getStorage(),
+ if (!HoodieHeartbeatClient.heartbeatExists(table.getStorage(),
config.getBasePath(), instantTimeToRollback)) {
throw new HoodieRollbackException(
"Found commits after time :" + instantTimeToRollback + ", please rollback greater commits first");
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
index dbf4ff45917..5dbfae67d5e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
@@ -90,7 +90,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
try {
HoodieTableMetaClient metaClient = table.getMetaClient();
List<String> partitionPaths =
- FSUtils.getAllPartitionPaths(context, table.getMetaClient().getBasePath(), false);
+ FSUtils.getAllPartitionPaths(context, table.getStorage(), table.getMetaClient().getBasePath(), false);
int numPartitions = Math.max(Math.min(partitionPaths.size(), config.getRollbackParallelism()), 1);
context.setJobStatus(this.getClass().getSimpleName(), "Creating Listing Rollback Plan: " + config.getTableName());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
index 1e0330a4def..b1ed3f2417a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
@@ -123,7 +123,8 @@ public class SavepointActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I
return latestFiles;
}));
} else {
- List<String> partitions = FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), table.getMetaClient().getBasePath());
+ List<String> partitions = FSUtils.getAllPartitionPaths(
+ context, table.getStorage(), config.getMetadataConfig(), table.getMetaClient().getBasePath());
latestFilesMap = context.mapToPair(partitions, partitionPath -> {
// Scan all partitions files with this commit time
LOG.info("Collecting latest files in partition path " + partitionPath);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/PartitionTTLStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/PartitionTTLStrategy.java
index 47768870930..00cdb4185f5 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/PartitionTTLStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/PartitionTTLStrategy.java
@@ -66,7 +66,8 @@ public abstract class PartitionTTLStrategy implements TTLStrategy, Serializable
List<String> partitionsForTTL;
if (StringUtils.isNullOrEmpty(partitionSelected)) {
// Return All partition paths
- partitionsForTTL = FSUtils.getAllPartitionPaths(hoodieTable.getContext(), writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ partitionsForTTL = FSUtils.getAllPartitionPaths(
+ hoodieTable.getContext(), hoodieTable.getStorage(), writeConfig.getMetadataConfig(), writeConfig.getBasePath());
} else {
partitionsForTTL = Arrays.asList(partitionSelected.split(","));
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java
index 65ee5639c8f..bdddef21c79 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java
@@ -71,7 +71,7 @@ public class DirectWriteMarkers extends WriteMarkers {
}
public DirectWriteMarkers(HoodieTable table, String instantTime) {
- this(table.getMetaClient().getStorage(),
+ this(table.getStorage(),
table.getMetaClient().getBasePath(),
table.getMetaClient().getMarkerFolderPath(instantTime),
instantTime);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java
index af1819f4cda..e3caacf402e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java
@@ -63,7 +63,7 @@ public class MarkerBasedRollbackUtils {
public static List<String> getAllMarkerPaths(HoodieTable table, HoodieEngineContext context,
String instant, int parallelism) throws IOException {
String markerDir = table.getMetaClient().getMarkerFolderPath(instant);
- HoodieStorage storage = table.getMetaClient().getStorage();
+ HoodieStorage storage = table.getStorage();
Option<MarkerType> markerTypeOption = readMarkerType(storage, markerDir);
// If there is no marker type file "MARKERS.type", first assume "DIRECT" markers are used.
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
index bff523f7f21..8a2cea2006c 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
@@ -23,16 +23,16 @@ import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ParquetUtils;
import org.apache.hudi.io.hadoop.HoodieAvroParquetWriter;
import org.apache.hudi.io.storage.HoodieParquetConfig;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
@@ -47,7 +47,6 @@ import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
-import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -57,7 +56,7 @@ public class TestHoodieAvroParquetWriter {
@Test
public void testProperWriting() throws IOException {
- StorageConfiguration<Configuration> storageConf = getDefaultStorageConf();
+ HoodieStorage storage = HoodieTestUtils.getStorage(tmpDir.toString());
HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(0xDEED);
List<GenericRecord> records = dataGen.generateGenericRecords(10);
@@ -71,7 +70,7 @@ public class TestHoodieAvroParquetWriter {
HoodieParquetConfig<HoodieAvroWriteSupport> parquetConfig =
new HoodieParquetConfig(writeSupport, CompressionCodecName.GZIP, ParquetWriter.DEFAULT_BLOCK_SIZE,
- ParquetWriter.DEFAULT_PAGE_SIZE, 1024 * 1024 * 1024, storageConf, 0.1, true);
+ ParquetWriter.DEFAULT_PAGE_SIZE, 1024 * 1024 * 1024, storage.getConf(), 0.1, true);
StoragePath filePath = new StoragePath(tmpDir.resolve("test.parquet").toAbsolutePath().toString());
@@ -85,7 +84,7 @@ public class TestHoodieAvroParquetWriter {
ParquetUtils utils = new ParquetUtils();
// Step 1: Make sure records are written appropriately
- List<GenericRecord> readRecords = utils.readAvroRecords(storageConf, filePath);
+ List<GenericRecord> readRecords = utils.readAvroRecords(storage, filePath);
assertEquals(toJson(records), toJson(readRecords));
@@ -95,8 +94,7 @@ public class TestHoodieAvroParquetWriter {
String minKey = recordKeys.stream().min(Comparator.naturalOrder()).get();
String maxKey = recordKeys.stream().max(Comparator.naturalOrder()).get();
- FileMetaData parquetMetadata = ParquetUtils.readMetadata(
- storageConf, filePath).getFileMetaData();
+ FileMetaData parquetMetadata = ParquetUtils.readMetadata(storage, filePath).getFileMetaData();
Map<String, String> extraMetadata = parquetMetadata.getKeyValueMetaData();
@@ -105,7 +103,7 @@ public class TestHoodieAvroParquetWriter {
assertEquals(extraMetadata.get(HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE), BloomFilterTypeCode.DYNAMIC_V0.name());
// Step 3: Make sure Bloom Filter contains all the record keys
- BloomFilter bloomFilter = utils.readBloomFilterFromMetadata(storageConf, filePath);
+ BloomFilter bloomFilter = utils.readBloomFilterFromMetadata(storage, filePath);
recordKeys.forEach(recordKey -> {
assertTrue(bloomFilter.mightContain(recordKey));
});
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java
index 4e938ef1cef..c156997441b 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java
@@ -21,7 +21,6 @@ package org.apache.hudi.metrics;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -42,7 +41,6 @@ public class TestHoodieConsoleMetrics {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -52,7 +50,7 @@ public class TestHoodieConsoleMetrics {
when(writeConfig.isMetricsOn()).thenReturn(true);
when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.CONSOLE);
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieGraphiteMetrics.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieGraphiteMetrics.java
index cf488405660..250bffa25aa 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieGraphiteMetrics.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieGraphiteMetrics.java
@@ -22,7 +22,6 @@ import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.NetworkTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
@@ -45,7 +44,6 @@ public class TestHoodieGraphiteMetrics {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -63,7 +61,7 @@ public class TestHoodieGraphiteMetrics {
when(metricsConfig.getGraphiteServerPort()).thenReturn(NetworkTestUtils.nextFreePort());
when(metricsConfig.getGraphiteReportPeriodSeconds()).thenReturn(30);
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
metrics.registerGauge("graphite_metric", 123L);
assertEquals("123", metrics.getRegistry().getGauges()
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
index 9daebd08661..20542cc4cc2 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
@@ -22,7 +22,6 @@ import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.NetworkTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -46,7 +45,6 @@ public class TestHoodieJmxMetrics {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -58,7 +56,7 @@ public class TestHoodieJmxMetrics {
when(metricsConfig.getJmxHost()).thenReturn("localhost");
when(metricsConfig.getJmxPort()).thenReturn(String.valueOf(NetworkTestUtils.nextFreePort()));
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
index 39cd0dc444f..83387a64a17 100755
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
@@ -25,7 +25,6 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.index.HoodieIndex;
-import org.apache.hudi.storage.StorageConfiguration;
import com.codahale.metrics.Timer;
import org.junit.jupiter.api.AfterEach;
@@ -51,7 +50,6 @@ public class TestHoodieMetrics {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -61,7 +59,7 @@ public class TestHoodieMetrics {
when(writeConfig.isMetricsOn()).thenReturn(true);
when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY);
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java
index 9a7b82b4485..8d57c909cd5 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java
@@ -18,13 +18,13 @@
package org.apache.hudi.metrics.datadog;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
-import org.apache.hudi.storage.StorageConfiguration;
import com.codahale.metrics.MetricRegistry;
import org.junit.jupiter.api.AfterEach;
@@ -48,8 +48,6 @@ public class TestDatadogMetricsReporter {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- @Mock
- StorageConfiguration storageConf;
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -73,7 +71,7 @@ public class TestDatadogMetricsReporter {
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
Throwable t = assertThrows(IllegalStateException.class, () -> {
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
});
assertEquals("Datadog cannot be initialized: API key is null or empty.", t.getMessage());
@@ -89,7 +87,7 @@ public class TestDatadogMetricsReporter {
when(metricsConfig.getDatadogMetricPrefix()).thenReturn("");
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
Throwable t = assertThrows(IllegalStateException.class, () -> {
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
});
assertEquals("Datadog cannot be initialized: Metric prefix is null or empty.", t.getMessage());
@@ -111,7 +109,7 @@ public class TestDatadogMetricsReporter {
when(metricsConfig.getMetricReporterMetricsNamePrefix()).thenReturn("");
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
assertDoesNotThrow(() -> {
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
});
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/m3/TestM3Metrics.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/m3/TestM3Metrics.java
index 954619f6174..76917c22f99 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/m3/TestM3Metrics.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/m3/TestM3Metrics.java
@@ -18,18 +18,13 @@
package org.apache.hudi.metrics.m3;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.when;
-
-import java.util.UUID;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.NetworkTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -37,6 +32,12 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.when;
+
@ExtendWith(MockitoExtension.class)
public class TestM3Metrics {
@@ -44,8 +45,6 @@ public class TestM3Metrics {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- @Mock
- StorageConfiguration storageConf;
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -66,7 +65,7 @@ public class TestM3Metrics {
when(metricsConfig.getM3Service()).thenReturn("hoodie");
when(metricsConfig.getM3Tags()).thenReturn("tag1=value1,tag2=value2");
when(metricsConfig.getMetricReporterMetricsNamePrefix()).thenReturn("");
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
metrics.registerGauge("metric1", 123L);
assertEquals("123", metrics.getRegistry().getGauges().get("metric1").getValue().toString());
@@ -84,7 +83,7 @@ public class TestM3Metrics {
when(metricsConfig.getM3Service()).thenReturn("hoodie");
when(metricsConfig.getM3Tags()).thenReturn("");
when(metricsConfig.getMetricReporterMetricsNamePrefix()).thenReturn("");
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
metrics.registerGauge("metric1", 123L);
assertEquals("123", metrics.getRegistry().getGauges().get("metric1").getValue().toString());
@@ -98,7 +97,7 @@ public class TestM3Metrics {
when(writeConfig.isMetricsOn()).thenReturn(true);
when(metricsConfig.getMetricReporterMetricsNamePrefix()).thenReturn("");
assertThrows(RuntimeException.class, () -> {
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
});
}
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java
index d95614a577a..e1aee8793f4 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java
@@ -24,7 +24,6 @@ import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
@@ -44,7 +43,6 @@ public class TestPrometheusReporter {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -63,8 +61,8 @@ public class TestPrometheusReporter {
when(metricsConfig.getPrometheusPort()).thenReturn(9090);
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
assertDoesNotThrow(() -> {
- new HoodieMetrics(writeConfig, storageConf);
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
});
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java
index c2c7695932d..20f8fe5e105 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java
@@ -26,7 +26,6 @@ import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.metrics.MetricUtils;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
@@ -62,7 +61,6 @@ public class TestPushGateWayReporter {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -81,7 +79,7 @@ public class TestPushGateWayReporter {
configureDefaultReporter();
assertDoesNotThrow(() -> {
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
});
@@ -106,7 +104,7 @@ public class TestPushGateWayReporter {
when(metricsConfig.getMetricReporterFileBasedConfigs()).thenReturn(propPrometheusPath + "," + propDatadogPath);
}
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
Map<String, Long> metricsMap = new HashMap<>();
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
index 6b7b967803f..3376ae37fbd 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
@@ -27,12 +27,13 @@ import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieValidationException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieIOFactory;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
@@ -145,9 +146,9 @@ public class GenericRecordValidationTestUtils {
public static Stream<GenericRecord> readHFile(Configuration conf, String[] paths) {
List<GenericRecord> valuesAsList = new LinkedList<>();
for (String path : paths) {
- StorageConfiguration storageConf = HadoopFSUtils.getStorageConf(conf);
+ HoodieStorage storage = new HoodieHadoopStorage(path, conf);
try (HoodieAvroHFileReaderImplBase reader = (HoodieAvroHFileReaderImplBase)
- HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
+ HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, new StoragePath(path), HoodieFileFormat.HFILE)) {
valuesAsList.addAll(HoodieAvroHFileReaderImplBase.readAllRecords(reader)
.stream().map(e -> (GenericRecord) e).collect(Collectors.toList()));
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java
index 409dfad159d..1d6bf73b2ec 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java
@@ -478,7 +478,7 @@ public class HoodieFlinkWriteClient<T> extends
case INSERT_OVERWRITE_TABLE:
Map<String, List<String>> partitionToExistingFileIds = new HashMap<>();
List<String> partitionPaths =
- FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), table.getMetaClient().getBasePath());
+ FSUtils.getAllPartitionPaths(context, table.getStorage(), config.getMetadataConfig(), table.getMetaClient().getBasePath());
if (partitionPaths != null && partitionPaths.size() > 0) {
context.setJobStatus(this.getClass().getSimpleName(), "Getting ExistingFileIds of all partitions: " + config.getTableName());
partitionToExistingFileIds = partitionPaths.stream().parallel()
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
index 15bbc7fa668..b08e814d15c 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
@@ -33,8 +33,8 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieInsertException;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
@@ -91,7 +91,7 @@ public class HoodieRowDataCreateHandle implements Serializable {
this.newRecordLocation = new HoodieRecordLocation(instantTime, fileId);
this.preserveHoodieMetadata = preserveHoodieMetadata;
this.currTimer = HoodieTimer.start();
- this.storage = table.getMetaClient().getStorage();
+ this.storage = table.getStorage();
this.path = makeNewPath(partitionPath);
this.writeStatus = new WriteStatus(table.shouldTrackSuccessRecords(),
@@ -175,8 +175,7 @@ public class HoodieRowDataCreateHandle implements Serializable {
stat.setFileId(fileId);
StoragePath storagePath = convertToStoragePath(path);
stat.setPath(new StoragePath(writeConfig.getBasePath()), storagePath);
- long fileSizeInBytes = FSUtils.getFileSize(
- table.getMetaClient().getStorage(), storagePath);
+ long fileSizeInBytes = FSUtils.getFileSize(table.getStorage(), storagePath);
stat.setTotalWriteBytes(fileSizeInBytes);
stat.setFileSizeInBytes(fileSizeInBytes);
stat.setTotalWriteErrors(writeStatus.getTotalErrorRecords());
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
index a61d0e566e5..d0734209f80 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
@@ -91,7 +91,7 @@ public class FlinkHoodieBackedTableMetadataWriter extends HoodieBackedTableMetad
protected void initRegistry() {
if (metadataWriteConfig.isMetricsOn()) {
// should support executor metrics
- this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), storageConf));
+ this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage()));
} else {
this.metrics = Option.empty();
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java
index 54c079b5166..90d12a2c496 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java
@@ -80,7 +80,7 @@ public class FlinkDeletePartitionCommitActionExecutor<T extends HoodieRecordPayl
// created requested
HoodieInstant dropPartitionsInstant =
new HoodieInstant(REQUESTED, REPLACE_COMMIT_ACTION, instantTime);
- if (!table.getMetaClient().getStorage().exists(new StoragePath(
+ if (!table.getStorage().exists(new StoragePath(
table.getMetaClient().getMetaPath(), dropPartitionsInstant.getFileName()))) {
HoodieRequestedReplaceMetadata requestedReplaceMetadata =
HoodieRequestedReplaceMetadata.newBuilder()
diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java
index 884d5125cfd..31640c5277c 100644
--- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java
+++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java
@@ -247,7 +247,7 @@ public class TestFlinkHoodieBloomIndex extends HoodieFlinkClientTestHarness {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).build();
List<Pair<String, Long>> results = HoodieIndexUtils.filterKeysFromFile(
- new StoragePath(java.nio.file.Paths.get(basePath, partition, filename).toString()), uuids, storageConf);
+ new StoragePath(java.nio.file.Paths.get(basePath, partition, filename).toString()), uuids, metaClient.getStorage());
assertEquals(results.size(), 2);
assertTrue(results.get(0).getLeft().equals("1eb5b87a-1feh-4edd-87b4-6ec96dc405a0")
|| results.get(1).getLeft().equals("1eb5b87a-1feh-4edd-87b4-6ec96dc405a0"));
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
index 6dd0fc09d72..5f0a3a2c64b 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
@@ -175,7 +175,7 @@ public abstract class JavaExecutionStrategy<T>
try {
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(config.getSchema()));
scanner = HoodieMergedLogRecordScanner.newBuilder()
- .withStorage(table.getMetaClient().getStorage())
+ .withStorage(table.getStorage())
.withBasePath(table.getMetaClient().getBasePath())
.withLogFilePaths(clusteringOp.getDeltaFilePaths())
.withReaderSchema(readerSchema)
@@ -192,7 +192,7 @@ public abstract class JavaExecutionStrategy<T>
baseFileReader = StringUtils.isNullOrEmpty(clusteringOp.getDataFilePath())
? Option.empty()
- : Option.of(HoodieIOFactory.getIOFactory(table.getStorageConf()).getReaderFactory(recordType)
+ : Option.of(HoodieIOFactory.getIOFactory(table.getStorage()).getReaderFactory(recordType)
.getFileReader(config, new StoragePath(clusteringOp.getDataFilePath())));
HoodieTableConfig tableConfig = table.getMetaClient().getTableConfig();
Iterator<HoodieRecord<T>> fileSliceReader = new HoodieFileSliceReader(baseFileReader, scanner, readerSchema, tableConfig.getPreCombineField(), writeConfig.getRecordMerger(),
@@ -221,7 +221,8 @@ public abstract class JavaExecutionStrategy<T>
private List<HoodieRecord<T>> readRecordsForGroupBaseFiles(List<ClusteringOperation> clusteringOps) {
List<HoodieRecord<T>> records = new ArrayList<>();
clusteringOps.forEach(clusteringOp -> {
- try (HoodieFileReader baseFileReader = HoodieIOFactory.getIOFactory(getHoodieTable().getStorageConf()).getReaderFactory(recordType)
+ try (HoodieFileReader baseFileReader = HoodieIOFactory.getIOFactory(getHoodieTable().getStorage())
+ .getReaderFactory(recordType)
.getFileReader(getHoodieTable().getConfig(), new StoragePath(clusteringOp.getDataFilePath()))) {
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(getWriteConfig().getSchema()));
Iterator<HoodieRecord> recordIterator = baseFileReader.getRecordIterator(readerSchema);
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
index 2f6e859e72c..aded8111b6e 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
@@ -79,7 +79,7 @@ public class JavaHoodieBackedTableMetadataWriter extends HoodieBackedTableMetada
@Override
protected void initRegistry() {
if (metadataWriteConfig.isMetricsOn()) {
- this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), storageConf));
+ this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage()));
} else {
this.metrics = Option.empty();
}
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertOverwriteTableCommitActionExecutor.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertOverwriteTableCommitActionExecutor.java
index 9d1eabf08db..f65e8a3d8b3 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertOverwriteTableCommitActionExecutor.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertOverwriteTableCommitActionExecutor.java
@@ -51,7 +51,7 @@ public class JavaInsertOverwriteTableCommitActionExecutor<T>
protected Map<String, List<String>> getPartitionToReplacedFileIds(HoodieWriteMetadata<List<WriteStatus>> writeResult) {
Map<String, List<String>> partitionToExistingFileIds = new HashMap<>();
List<String> partitionPaths = FSUtils.getAllPartitionPaths(context,
- table.getMetaClient().getBasePath(), config.isMetadataTableEnabled());
+ table.getStorage(), table.getMetaClient().getBasePath(), config.isMetadataTableEnabled());
if (partitionPaths != null && partitionPaths.size() > 0) {
partitionToExistingFileIds = context.mapToPair(partitionPaths,
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
index c5be86e7405..f82725868e8 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
@@ -172,7 +172,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storageConf, new StoragePath(filePath.toUri()));
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storage, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records1) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
@@ -204,7 +204,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
records1.addAll(records2);
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
int index = 0;
for (GenericRecord record : fileRecords) {
assertEquals(records1.get(index).getRecordKey(), record.get("_row_key").toString());
@@ -239,7 +239,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storageConf, new StoragePath(filePath.toUri()));
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storage, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records1) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
@@ -260,7 +260,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
records1.addAll(records2);
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
assertEquals(fileRecords.size(), mergeAllowDuplicateOnInsertsEnable ? records1.size() : records2.size());
int index = 0;
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
index c2b85bd70b0..1e6e74664d8 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
@@ -549,7 +549,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
List<FileSlice> fileSlices = table.getSliceView().getLatestFileSlices("files").collect(Collectors.toList());
HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
- HoodieIOFactory.getIOFactory(context.getStorageConf()).getReaderFactory(HoodieRecordType.AVRO)
+ HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(writeConfig, new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
records.forEach(entry -> {
@@ -983,7 +983,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
final HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
- HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecordType.AVRO)
+ HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(table.getConfig(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
records.forEach(entry -> {
@@ -2393,7 +2393,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
assertNoWriteErrors(writeStatuses);
validateMetadata(client);
- Metrics metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storageConf);
+ Metrics metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storage);
assertTrue(metrics.getRegistry().getGauges().containsKey(HoodieMetadataMetrics.INITIALIZE_STR + ".count"));
assertTrue(metrics.getRegistry().getGauges().containsKey(HoodieMetadataMetrics.INITIALIZE_STR + ".totalDuration"));
assertTrue((Long) metrics.getRegistry().getGauges().get(HoodieMetadataMetrics.INITIALIZE_STR + ".count").getValue() >= 1L);
@@ -2706,7 +2706,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// Partitions should match
FileSystemBackedTableMetadata fsBackedTableMetadata = new FileSystemBackedTableMetadata(
- engineContext, metaClient.getTableConfig(), storageConf.newInstance(), config.getBasePath());
+ engineContext, metaClient.getTableConfig(), metaClient.getStorage(), config.getBasePath());
List<String> fsPartitions = fsBackedTableMetadata.getAllPartitionPaths();
List<String> metadataPartitions = tableMetadata.getAllPartitionPaths();
@@ -2819,7 +2819,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// Metadata table has a fixed number of partitions
// Cannot use FSUtils.getAllFoldersWithPartitionMetaFile for this as that function filters all directory
// in the .hoodie folder.
- List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, getMetadataTableBasePath(basePath), false);
+ List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, storage, getMetadataTableBasePath(basePath), false);
assertEquals(metadataWriter.getEnabledPartitionTypes().size(), metadataTablePartitions.size());
final Map<String, MetadataPartitionType> metadataEnabledPartitionTypes = new HashMap<>();
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
index 233b1592a6b..a6204ca75e6 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
@@ -1025,7 +1025,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
private Set<String> verifyRecordKeys(List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, List<GenericRecord> records) {
for (WriteStatus status : allStatus) {
StoragePath filePath = new StoragePath(basePath, status.getStat().getPath());
- records.addAll(getFileUtilsInstance(metaClient).readAvroRecords(storageConf, filePath));
+ records.addAll(getFileUtilsInstance(metaClient).readAvroRecords(storage, filePath));
}
Set<String> expectedKeys = recordsToRecordKeySet(expectedRecords);
assertEquals(records.size(), expectedKeys.size());
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
index f9262c7f939..2e95ca785ef 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
@@ -161,13 +161,13 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storageConf, new StoragePath(filePath.toUri()));
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storage, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
GenericRecord newRecord;
int index = 0;
for (GenericRecord record : fileRecords) {
@@ -202,7 +202,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
// Check whether the record has been updated
Path updatedFilePath = allFiles[0].getPath();
BloomFilter updatedFilter =
- fileUtils.readBloomFilterFromMetadata(storageConf, new StoragePath(updatedFilePath.toUri()));
+ fileUtils.readBloomFilterFromMetadata(storage, new StoragePath(updatedFilePath.toUri()));
for (HoodieRecord record : records) {
// No change to the _row_key
assertTrue(updatedFilter.mightContain(record.getRecordKey()));
@@ -508,13 +508,13 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storageConf, new StoragePath(filePath.toUri()));
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storage, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
int index = 0;
for (GenericRecord record : fileRecords) {
assertEquals(records.get(index).getRecordKey(), record.get("_row_key").toString());
@@ -533,7 +533,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
filePath = allFiles[0].getPath();
// Read the base file, check the record content
- fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
// Check that the two records are deleted successfully
assertEquals(1, fileRecords.size());
assertEquals(records.get(1).getRecordKey(), fileRecords.get(0).get("_row_key").toString());
@@ -550,7 +550,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
filePath = allFiles[0].getPath();
// Read the base file, check the record content
- fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
// Check whether all records have been deleted
assertEquals(0, fileRecords.size());
}
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
index eda3a8818b6..ba1d71845f4 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
@@ -260,7 +260,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
}
protected HoodieTableMetadata metadata(HoodieWriteConfig clientConfig, HoodieEngineContext engineContext) {
- return HoodieTableMetadata.create(engineContext, clientConfig.getMetadataConfig(), clientConfig.getBasePath());
+ return HoodieTableMetadata.create(engineContext, metaClient.getStorage(), clientConfig.getMetadataConfig(), clientConfig.getBasePath());
}
/**
@@ -405,7 +405,8 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
// Metadata table has a fixed number of partitions
// Cannot use FSUtils.getAllFoldersWithPartitionMetaFile for this as that function filters all directory
// in the .hoodie folder.
- List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, HoodieTableMetadata.getMetadataTableBasePath(basePath), false);
+ List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(
+ engineContext, storage, HoodieTableMetadata.getMetadataTableBasePath(basePath), false);
List<MetadataPartitionType> enabledPartitionTypes = metadataWriter.getEnabledPartitionTypes();
@@ -908,7 +909,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
HashMap<String, String> paths =
getLatestFileIDsToFullPath(basePath, commitTimeline, Arrays.asList(commitInstant));
return paths.values().stream().map(StoragePath::new).flatMap(path ->
- FileFormatUtils.getInstance(path).readAvroRecords(context.getStorageConf(), path).stream())
+ FileFormatUtils.getInstance(path).readAvroRecords(storage, path).stream())
.filter(record -> {
if (filterByCommitTime) {
Object commitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD);
@@ -938,7 +939,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
List<HoodieBaseFile> latestFiles = getLatestBaseFiles(basePath, storage, paths);
return latestFiles.stream().mapToLong(baseFile ->
FileFormatUtils.getInstance(baseFile.getStoragePath())
- .readAvroRecords(context.getStorageConf(), baseFile.getStoragePath()).size())
+ .readAvroRecords(storage, baseFile.getStoragePath()).size())
.sum();
} catch (Exception e) {
throw new HoodieException("Error reading hoodie table as a dataframe", e);
@@ -975,7 +976,8 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
HashMap<String, String> fileIdToFullPath = getLatestFileIDsToFullPath(basePath, commitTimeline, commitsToReturn);
String[] paths = fileIdToFullPath.values().toArray(new String[fileIdToFullPath.size()]);
if (paths[0].endsWith(HoodieFileFormat.PARQUET.getFileExtension())) {
- return Arrays.stream(paths).map(StoragePath::new).flatMap(path -> FileFormatUtils.getInstance(path).readAvroRecords(context.getStorageConf(), path).stream())
+ return Arrays.stream(paths).map(StoragePath::new).flatMap(path -> FileFormatUtils.getInstance(path)
+ .readAvroRecords(storage, path).stream())
.filter(record -> {
if (lastCommitTimeOpt.isPresent()) {
Object commitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
index 1bbd5a73401..75b42491eda 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
@@ -58,8 +58,10 @@ import org.apache.hudi.io.IOUtils;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.HoodieWriteMetadata;
@@ -307,7 +309,7 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
try {
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(config.getSchema()));
HoodieMergedLogRecordScanner scanner = HoodieMergedLogRecordScanner.newBuilder()
- .withStorage(table.getMetaClient().getStorage())
+ .withStorage(table.getStorage())
.withBasePath(table.getMetaClient().getBasePath())
.withLogFilePaths(clusteringOp.getDeltaFilePaths())
.withReaderSchema(readerSchema)
@@ -385,8 +387,10 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
private HoodieFileReader getBaseOrBootstrapFileReader(StorageConfiguration<?> storageConf, String bootstrapBasePath, Option<String[]> partitionFields, ClusteringOperation clusteringOp)
throws IOException {
- HoodieFileReader baseFileReader = getHoodieSparkIOFactory(storageConf).getReaderFactory(recordType)
- .getFileReader(writeConfig, new StoragePath(clusteringOp.getDataFilePath()));
+ StoragePath dataFilePath = new StoragePath(clusteringOp.getDataFilePath());
+ HoodieStorage storage = new HoodieHadoopStorage(dataFilePath, storageConf);
+ HoodieFileReader baseFileReader = getHoodieSparkIOFactory(storage).getReaderFactory(recordType)
+ .getFileReader(writeConfig, dataFilePath);
// handle bootstrap path
if (StringUtils.nonEmpty(clusteringOp.getBootstrapFilePath()) && StringUtils.nonEmpty(bootstrapBasePath)) {
String bootstrapFilePath = clusteringOp.getBootstrapFilePath();
@@ -397,9 +401,9 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
partitionValues = getPartitionFieldVals(partitionFields, partitionFilePath, bootstrapBasePath, baseFileReader.getSchema(),
storageConf.unwrapAs(Configuration.class));
}
- baseFileReader = getHoodieSparkIOFactory(storageConf).getReaderFactory(recordType).newBootstrapFileReader(
+ baseFileReader = getHoodieSparkIOFactory(storage).getReaderFactory(recordType).newBootstrapFileReader(
baseFileReader,
- getHoodieSparkIOFactory(storageConf).getReaderFactory(recordType).getFileReader(
+ getHoodieSparkIOFactory(storage).getReaderFactory(recordType).getFileReader(
writeConfig, new StoragePath(bootstrapFilePath)), partitionFields,
partitionValues);
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
index a7faca1a418..b0ab5ee7767 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
@@ -147,7 +147,8 @@ public abstract class SingleSparkJobExecutionStrategy<T>
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(getWriteConfig().getSchema()));
Iterable<HoodieRecord<T>> indexedRecords = () -> {
try {
- HoodieFileReader baseFileReader = getHoodieSparkIOFactory(getHoodieTable().getStorageConf()).getReaderFactory(recordType)
+ HoodieFileReader baseFileReader = getHoodieSparkIOFactory(getHoodieTable().getStorage())
+ .getReaderFactory(recordType)
.getFileReader(writeConfig, new StoragePath(clusteringOp.getDataFilePath()));
Option<BaseKeyGenerator> keyGeneratorOp = HoodieSparkKeyGeneratorFactory.createBaseKeyGenerator(writeConfig);
// NOTE: Record have to be cloned here to make sure if it holds low-level engine-specific
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java
index 25fae3cb6f5..95055e582bc 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java
@@ -59,7 +59,7 @@ public abstract class SparkPreCommitValidator<T, I, K, O extends HoodieData<Writ
this.table = table;
this.engineContext = engineContext;
this.writeConfig = writeConfig;
- this.metrics = new HoodieMetrics(writeConfig, engineContext.getStorageConf());
+ this.metrics = new HoodieMetrics(writeConfig, table.getStorage());
}
protected Set<String> getPartitionsModified(HoodieWriteMetadata<O> writeResult) {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
index ae3cb76e33d..0350fcd331f 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
@@ -28,6 +28,7 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.index.HoodieIndexUtils;
import org.apache.hudi.io.HoodieKeyLookupResult;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.spark.api.java.function.FlatMapFunction;
@@ -127,7 +128,7 @@ public class HoodieFileProbingFunction implements
final HoodieBaseFile dataFile = fileIDBaseFileMap.get(fileId);
List<Pair<String, Long>> matchingKeysAndPositions = HoodieIndexUtils.filterKeysFromFile(
- dataFile.getStoragePath(), candidateRecordKeys, storageConf);
+ dataFile.getStoragePath(), candidateRecordKeys, HoodieStorageUtils.getStorage(dataFile.getStoragePath(), storageConf));
LOG.debug(
String.format("Bloom filter candidates (%d) / false positives (%d), actual matches (%d)",
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java
index 06b33c8dded..bc3538429e7 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java
@@ -21,7 +21,7 @@ package org.apache.hudi.io.storage;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -31,21 +31,21 @@ import java.io.IOException;
public class HoodieSparkFileReaderFactory extends HoodieFileReaderFactory {
- public HoodieSparkFileReaderFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieSparkFileReaderFactory(HoodieStorage storage) {
+ super(storage);
}
@Override
public HoodieFileReader newParquetFileReader(StoragePath path) {
- storageConf.setIfUnset(SQLConf.PARQUET_BINARY_AS_STRING().key(), SQLConf.PARQUET_BINARY_AS_STRING().defaultValueString());
- storageConf.setIfUnset(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), SQLConf.PARQUET_INT96_AS_TIMESTAMP().defaultValueString());
- storageConf.setIfUnset(SQLConf.CASE_SENSITIVE().key(), SQLConf.CASE_SENSITIVE().defaultValueString());
+ storage.getConf().setIfUnset(SQLConf.PARQUET_BINARY_AS_STRING().key(), SQLConf.PARQUET_BINARY_AS_STRING().defaultValueString());
+ storage.getConf().setIfUnset(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), SQLConf.PARQUET_INT96_AS_TIMESTAMP().defaultValueString());
+ storage.getConf().setIfUnset(SQLConf.CASE_SENSITIVE().key(), SQLConf.CASE_SENSITIVE().defaultValueString());
// Using string value of this conf to preserve compatibility across spark versions.
- storageConf.setIfUnset("spark.sql.legacy.parquet.nanosAsLong", "false");
+ storage.getConf().setIfUnset("spark.sql.legacy.parquet.nanosAsLong", "false");
// This is a required config since Spark 3.4.0: SQLConf.PARQUET_INFER_TIMESTAMP_NTZ_ENABLED
// Using string value of this conf to preserve compatibility across spark versions.
- storageConf.setIfUnset("spark.sql.parquet.inferTimestampNTZ.enabled", "true");
- return new HoodieSparkParquetReader(storageConf, path);
+ storage.getConf().setIfUnset("spark.sql.parquet.inferTimestampNTZ.enabled", "true");
+ return new HoodieSparkParquetReader(storage, path);
}
@Override
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java
index 6a513e2d7d6..eedc560bdaf 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java
@@ -27,6 +27,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.storage.row.HoodieRowParquetConfig;
import org.apache.hudi.io.storage.row.HoodieRowParquetWriteSupport;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
@@ -42,8 +43,8 @@ import java.io.OutputStream;
public class HoodieSparkFileWriterFactory extends HoodieFileWriterFactory {
- public HoodieSparkFileWriterFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieSparkFileWriterFactory(HoodieStorage storage) {
+ super(storage);
}
@Override
@@ -56,14 +57,14 @@ public class HoodieSparkFileWriterFactory extends HoodieFileWriterFactory {
if (compressionCodecName.isEmpty()) {
compressionCodecName = null;
}
- HoodieRowParquetWriteSupport writeSupport = getHoodieRowParquetWriteSupport(storageConf, schema,
+ HoodieRowParquetWriteSupport writeSupport = getHoodieRowParquetWriteSupport(storage.getConf(), schema,
config, enableBloomFilter(populateMetaFields, config));
HoodieRowParquetConfig parquetConfig = new HoodieRowParquetConfig(writeSupport,
CompressionCodecName.fromConf(compressionCodecName),
config.getIntOrDefault(HoodieStorageConfig.PARQUET_BLOCK_SIZE),
config.getIntOrDefault(HoodieStorageConfig.PARQUET_PAGE_SIZE),
config.getLongOrDefault(HoodieStorageConfig.PARQUET_MAX_FILE_SIZE),
- storageConf.unwrapAs(Configuration.class),
+ storage.getConf().unwrapAs(Configuration.class),
config.getDoubleOrDefault(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION),
config.getBooleanOrDefault(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED));
parquetConfig.getHadoopConf().addResource(writeSupport.getHadoopConf());
@@ -74,7 +75,7 @@ public class HoodieSparkFileWriterFactory extends HoodieFileWriterFactory {
protected HoodieFileWriter newParquetFileWriter(OutputStream outputStream, HoodieConfig config,
Schema schema) throws IOException {
boolean enableBloomFilter = false;
- HoodieRowParquetWriteSupport writeSupport = getHoodieRowParquetWriteSupport(storageConf, schema, config, enableBloomFilter);
+ HoodieRowParquetWriteSupport writeSupport = getHoodieRowParquetWriteSupport(storage.getConf(), schema, config, enableBloomFilter);
String compressionCodecName = config.getStringOrDefault(HoodieStorageConfig.PARQUET_COMPRESSION_CODEC_NAME);
// Support PARQUET_COMPRESSION_CODEC_NAME is ""
if (compressionCodecName.isEmpty()) {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java
index 9d673b98908..55a0aad05f2 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java
@@ -20,25 +20,26 @@
package org.apache.hudi.io.storage;
import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.io.hadoop.HoodieHadoopIOFactory;
+import org.apache.hudi.storage.HoodieStorage;
/**
* Creates readers and writers for SPARK and AVRO record payloads
*/
public class HoodieSparkIOFactory extends HoodieHadoopIOFactory {
- public HoodieSparkIOFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieSparkIOFactory(HoodieStorage storage) {
+ super(storage);
}
- public static HoodieSparkIOFactory getHoodieSparkIOFactory(StorageConfiguration<?> storageConf) {
- return new HoodieSparkIOFactory(storageConf);
+ public static HoodieSparkIOFactory getHoodieSparkIOFactory(HoodieStorage storage) {
+ return new HoodieSparkIOFactory(storage);
}
@Override
public HoodieFileReaderFactory getReaderFactory(HoodieRecord.HoodieRecordType recordType) {
if (recordType == HoodieRecord.HoodieRecordType.SPARK) {
- return new HoodieSparkFileReaderFactory(storageConf);
+ return new HoodieSparkFileReaderFactory(storage);
}
return super.getReaderFactory(recordType);
}
@@ -46,7 +47,7 @@ public class HoodieSparkIOFactory extends HoodieHadoopIOFactory {
@Override
public HoodieFileWriterFactory getWriterFactory(HoodieRecord.HoodieRecordType recordType) {
if (recordType == HoodieRecord.HoodieRecordType.SPARK) {
- return new HoodieSparkFileWriterFactory(storageConf);
+ return new HoodieSparkFileWriterFactory(storage);
}
return super.getWriterFactory(recordType);
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java
index 7203eb8a572..4dc60c07283 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java
@@ -31,7 +31,7 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.common.util.collection.CloseableMappingIterator;
import org.apache.hudi.common.util.collection.Pair;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -60,31 +60,31 @@ import static org.apache.parquet.avro.AvroSchemaConverter.ADD_LIST_ELEMENT_RECOR
public class HoodieSparkParquetReader implements HoodieSparkFileReader {
private final StoragePath path;
- private final StorageConfiguration<?> conf;
+ private final HoodieStorage storage;
private final FileFormatUtils parquetUtils;
private List<ParquetReaderIterator> readerIterators = new ArrayList<>();
- public HoodieSparkParquetReader(StorageConfiguration<?> conf, StoragePath path) {
+ public HoodieSparkParquetReader(HoodieStorage storage, StoragePath path) {
this.path = path;
- this.conf = conf.newInstance();
+ this.storage = storage.newInstance(path, storage.getConf().newInstance());
// Avoid adding record in list element when convert parquet schema to avro schema
- conf.set(ADD_LIST_ELEMENT_RECORDS, "false");
+ this.storage.getConf().set(ADD_LIST_ELEMENT_RECORDS, "false");
this.parquetUtils = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET);
}
@Override
public String[] readMinMaxRecordKeys() {
- return parquetUtils.readMinMaxRecordKeys(conf, path);
+ return parquetUtils.readMinMaxRecordKeys(storage, path);
}
@Override
public BloomFilter readBloomFilter() {
- return parquetUtils.readBloomFilterFromMetadata(conf, path);
+ return parquetUtils.readBloomFilterFromMetadata(storage, path);
}
@Override
public Set<Pair<String, Long>> filterRowKeys(Set<String> candidateRowKeys) {
- return parquetUtils.filterRowKeys(conf, path, candidateRowKeys);
+ return parquetUtils.filterRowKeys(storage, path, candidateRowKeys);
}
@Override
@@ -123,12 +123,12 @@ public class HoodieSparkParquetReader implements HoodieSparkFileReader {
}
StructType readerStructType = HoodieInternalRowUtils.getCachedSchema(readerSchema);
StructType requestedStructType = HoodieInternalRowUtils.getCachedSchema(requestedSchema);
- conf.set(ParquetReadSupport.PARQUET_READ_SCHEMA, readerStructType.json());
- conf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA(), requestedStructType.json());
- conf.set(SQLConf.PARQUET_BINARY_AS_STRING().key(), SQLConf.get().getConf(SQLConf.PARQUET_BINARY_AS_STRING()).toString());
- conf.set(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), SQLConf.get().getConf(SQLConf.PARQUET_INT96_AS_TIMESTAMP()).toString());
+ storage.getConf().set(ParquetReadSupport.PARQUET_READ_SCHEMA, readerStructType.json());
+ storage.getConf().set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA(), requestedStructType.json());
+ storage.getConf().set(SQLConf.PARQUET_BINARY_AS_STRING().key(), SQLConf.get().getConf(SQLConf.PARQUET_BINARY_AS_STRING()).toString());
+ storage.getConf().set(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), SQLConf.get().getConf(SQLConf.PARQUET_INT96_AS_TIMESTAMP()).toString());
ParquetReader<InternalRow> reader = ParquetReader.<InternalRow>builder((ReadSupport) new ParquetReadSupport(), new Path(path.toUri()))
- .withConf(conf.unwrapAs(Configuration.class))
+ .withConf(storage.getConf().unwrapAs(Configuration.class))
.build();
ParquetReaderIterator<InternalRow> parquetReaderIterator = new ParquetReaderIterator<>(reader);
readerIterators.add(parquetReaderIterator);
@@ -140,8 +140,8 @@ public class HoodieSparkParquetReader implements HoodieSparkFileReader {
// Some types in avro are not compatible with parquet.
// Avro only supports representing Decimals as fixed byte array
// and therefore if we convert to Avro directly we'll lose logical type-info.
- MessageType messageType = ((ParquetUtils) parquetUtils).readSchema(conf, path);
- StructType structType = new ParquetToSparkSchemaConverter(conf.unwrapAs(Configuration.class)).convert(messageType);
+ MessageType messageType = ((ParquetUtils) parquetUtils).readSchema(storage, path);
+ StructType structType = new ParquetToSparkSchemaConverter(storage.getConf().unwrapAs(Configuration.class)).convert(messageType);
return SparkAdapterSupport$.MODULE$.sparkAdapter()
.getAvroSchemaConverters()
.toAvroType(structType, true, messageType.getName(), StringUtils.EMPTY_STRING);
@@ -154,6 +154,6 @@ public class HoodieSparkParquetReader implements HoodieSparkFileReader {
@Override
public long getTotalRecords() {
- return parquetUtils.getRowCount(conf, path);
+ return parquetUtils.getRowCount(storage, path);
}
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
index 2a8c395d0d5..fb102a36bcb 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
@@ -112,7 +112,7 @@ public class HoodieRowCreateHandle implements Serializable {
this.currTimer = HoodieTimer.start();
- HoodieStorage storage = table.getMetaClient().getStorage();
+ HoodieStorage storage = table.getStorage();
String writeToken = getWriteToken(taskPartitionId, taskId, taskEpochId);
String fileName = FSUtils.makeBaseFileName(instantTime, writeToken, this.fileId,
@@ -238,7 +238,7 @@ public class HoodieRowCreateHandle implements Serializable {
stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT);
stat.setFileId(fileId);
stat.setPath(new StoragePath(writeConfig.getBasePath()), path);
- long fileSizeInBytes = FSUtils.getFileSize(table.getMetaClient().getStorage(), path);
+ long fileSizeInBytes = FSUtils.getFileSize(table.getStorage(), path);
stat.setTotalWriteBytes(fileSizeInBytes);
stat.setFileSizeInBytes(fileSizeInBytes);
stat.setTotalWriteErrors(writeStatus.getTotalErrorRecords());
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
index 84d282ef51c..32220388972 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
@@ -121,7 +121,7 @@ public class SparkHoodieBackedTableMetadataWriter extends HoodieBackedTableMetad
} else {
registry = Registry.getRegistry("HoodieMetadata");
}
- this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), storageConf));
+ this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage()));
} else {
this.metrics = Option.empty();
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java
index 565551505c6..e1d3d803ed0 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java
@@ -82,7 +82,7 @@ class ParquetBootstrapMetadataHandler extends BaseBootstrapMetadataHandler {
Schema schema) throws Exception {
HoodieRecord.HoodieRecordType recordType = table.getConfig().getRecordMerger().getRecordType();
- HoodieFileReader reader = getHoodieSparkIOFactory(table.getStorageConf()).getReaderFactory(recordType)
+ HoodieFileReader reader = getHoodieSparkIOFactory(table.getStorage()).getReaderFactory(recordType)
.getFileReader(table.getConfig(), sourceFilePath);
HoodieExecutor<Void> executor = null;
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
index c51bb5f21c4..843581a36a0 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
@@ -77,7 +77,7 @@ public class SparkDeletePartitionCommitActionExecutor<T>
// created requested
HoodieInstant dropPartitionsInstant =
new HoodieInstant(REQUESTED, REPLACE_COMMIT_ACTION, instantTime);
- if (!table.getMetaClient().getStorage().exists(
+ if (!table.getStorage().exists(
new StoragePath(table.getMetaClient().getMetaPath(),
dropPartitionsInstant.getFileName()))) {
HoodieRequestedReplaceMetadata requestedReplaceMetadata =
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java
index 868c8e2b42a..776ec1dbf1e 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java
@@ -45,7 +45,8 @@ public class SparkInsertOverwriteTableCommitActionExecutor<T>
@Override
protected Map<String, List<String>> getPartitionToReplacedFileIds(HoodieWriteMetadata<HoodieData<WriteStatus>> writeMetadata) {
- List<String> partitionPaths = FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), table.getMetaClient().getBasePath());
+ List<String> partitionPaths = FSUtils.getAllPartitionPaths(
+ context, table.getStorage(), config.getMetadataConfig(), table.getMetaClient().getBasePath());
if (partitionPaths == null || partitionPaths.isEmpty()) {
return Collections.emptyMap();
}
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala
index d9d33dbf1d4..c3d49ca47a8 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala
@@ -24,7 +24,7 @@ import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.util.ValidationUtils.checkState
import org.apache.hudi.common.util.collection.{ClosableIterator, CloseableMappingIterator}
import org.apache.hudi.io.storage.{HoodieSparkFileReaderFactory, HoodieSparkParquetReader}
-import org.apache.hudi.storage.{StorageConfiguration, StoragePath}
+import org.apache.hudi.storage.{HoodieStorage, StoragePath}
import org.apache.hudi.util.CloseableInternalRowIterator
import org.apache.avro.Schema
@@ -54,12 +54,7 @@ class SparkFileFormatInternalRowReaderContext(readerMaps: mutable.Map[Long, Part
lazy val sparkAdapter = SparkAdapterSupport.sparkAdapter
val deserializerMap: mutable.Map[Schema, HoodieAvroDeserializer] = mutable.Map()
- override def getFileRecordIterator(filePath: StoragePath,
- start: Long,
- length: Long,
- dataSchema: Schema,
- requiredSchema: Schema,
- conf: StorageConfiguration[_]): ClosableIterator[InternalRow] = {
+ override def getFileRecordIterator(filePath: StoragePath, start: Long, length: Long, dataSchema: Schema, requiredSchema: Schema, storage: HoodieStorage): ClosableIterator[InternalRow] = {
// partition value is empty because the spark parquet reader will append the partition columns to
// each row if they are given. That is the only usage of the partition values in the reader.
val fileInfo = sparkAdapter.getSparkPartitionedFileUtils
@@ -68,7 +63,7 @@ class SparkFileFormatInternalRowReaderContext(readerMaps: mutable.Map[Long, Part
val structType: StructType = HoodieInternalRowUtils.getCachedSchema(requiredSchema)
val projection: UnsafeProjection = HoodieInternalRowUtils.getCachedUnsafeProjection(structType, structType)
new CloseableMappingIterator[InternalRow, UnsafeRow](
- new HoodieSparkFileReaderFactory(conf).newParquetFileReader(filePath)
+ new HoodieSparkFileReaderFactory(storage).newParquetFileReader(filePath)
.asInstanceOf[HoodieSparkParquetReader].getInternalRowIterator(dataSchema, requiredSchema),
new java.util.function.Function[InternalRow, UnsafeRow] {
override def apply(data: InternalRow): UnsafeRow = {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
index acd6b24cb87..2a045ac05c9 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
@@ -136,7 +136,7 @@ public class TestClientRollback extends HoodieClientTestBase {
assertNoWriteErrors(statuses);
HoodieWriteConfig config = getConfig();
List<String> partitionPaths =
- FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), cfg.getBasePath());
+ FSUtils.getAllPartitionPaths(context, storage, config.getMetadataConfig(), cfg.getBasePath());
metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieSparkTable table = HoodieSparkTable.create(getConfig(), context, metaClient);
final BaseFileOnlyView view1 = table.getBaseFileOnlyView();
@@ -318,7 +318,7 @@ public class TestClientRollback extends HoodieClientTestBase {
assertNoWriteErrors(statuses);
HoodieWriteConfig config = getConfig();
List<String> partitionPaths =
- FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), cfg.getBasePath());
+ FSUtils.getAllPartitionPaths(context, storage, config.getMetadataConfig(), cfg.getBasePath());
metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieSparkTable table = HoodieSparkTable.create(getConfig(), context, metaClient);
final BaseFileOnlyView view1 = table.getBaseFileOnlyView();
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java
index f115b7c7202..d8b7710d5c8 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java
@@ -135,7 +135,7 @@ public class TestUpdateSchemaEvolution extends HoodieSparkClientTestHarness impl
HoodieMergeHandle mergeHandle = new HoodieMergeHandle(updateTable.getConfig(), "101", updateTable,
updateRecords.iterator(), updateRecords.get(0).getPartitionPath(), insertResult.getFileId(), supplier, Option.empty());
List<GenericRecord> oldRecords = FileFormatUtils.getInstance(updateTable.getBaseFileFormat())
- .readAvroRecords(updateTable.getStorageConf(),
+ .readAvroRecords(updateTable.getStorage(),
new StoragePath(updateTable.getConfig().getBasePath() + "/" + insertResult.getStat().getPath()),
mergeHandle.getWriterSchemaWithMetaFields());
for (GenericRecord rec : oldRecords) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
index 039be9048a2..ffc8d8319f1 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
@@ -173,7 +173,8 @@ public class TestExternalPathHandling extends HoodieClientTestBase {
assertFileGroupCorrectness(instantTime3, partitionPath2, filePath3, fileId3, partitionPath2.isEmpty() ? 2 : 1);
// assert that column stats are correct
- HoodieBackedTableMetadata hoodieBackedTableMetadata = new HoodieBackedTableMetadata(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), true);
+ HoodieBackedTableMetadata hoodieBackedTableMetadata = new HoodieBackedTableMetadata(
+ context, metaClient.getStorage(), writeConfig.getMetadataConfig(), writeConfig.getBasePath(), true);
assertEmptyColStats(hoodieBackedTableMetadata, partitionPath1, fileName1);
assertColStats(hoodieBackedTableMetadata, partitionPath1, fileName2);
assertColStats(hoodieBackedTableMetadata, partitionPath2, fileName3);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
index 4da78d84980..35ffd1035e1 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
@@ -767,7 +767,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
metadataWriter.deletePartitions("0000003", Arrays.asList(COLUMN_STATS));
HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
- List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, metadataMetaClient.getBasePath(), false);
+ List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(
+ engineContext, metadataMetaClient.getStorage(), metadataMetaClient.getBasePath(), false);
// partition should be physically deleted
assertEquals(metadataWriter.getEnabledPartitionTypes().size(), metadataTablePartitions.size());
assertFalse(metadataTablePartitions.contains(COLUMN_STATS.getPartitionPath()));
@@ -826,7 +827,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
List<FileSlice> fileSlices = table.getSliceView().getLatestFileSlices("files").collect(Collectors.toList());
HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
- getHoodieSparkIOFactory(context.getStorageConf()).getReaderFactory(HoodieRecordType.AVRO).getFileReader(
+ getHoodieSparkIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO).getFileReader(
table.getConfig(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
records.forEach(entry -> {
@@ -1141,7 +1142,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
HoodieTableMetadata metadataReader = HoodieTableMetadata.create(
- context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
Map<String, List<HoodieRecordGlobalLocation>> result = metadataReader
.readRecordIndex(records1.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toList()));
assertEquals(0, result.size(), "RI should not return entries that are rolled back.");
@@ -1449,7 +1450,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
final HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
- HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase) getHoodieSparkIOFactory(context.getStorageConf())
+ HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase) getHoodieSparkIOFactory(storage)
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(table.getConfig(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
@@ -3041,7 +3042,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
doWriteOperation(testTable, InProcessTimeGenerator.createNewInstantTime(1));
doWriteOperation(testTable, InProcessTimeGenerator.createNewInstantTime(1));
// ensure the compaction is triggered and executed
- try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), true)) {
+ try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(
+ context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), true)) {
HoodieTableMetaClient metadataMetaClient = metadata.getMetadataMetaClient();
final HoodieActiveTimeline activeTimeline = metadataMetaClient.reloadActiveTimeline();
Option<HoodieInstant> lastCompaction = activeTimeline.filterCompletedInstants()
@@ -3114,7 +3116,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertNoWriteErrors(writeStatuses);
validateMetadata(client);
- Metrics metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storageConf);
+ Metrics metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storage);
assertTrue(metrics.getRegistry().getGauges().containsKey(HoodieMetadataMetrics.INITIALIZE_STR + ".count"));
assertTrue(metrics.getRegistry().getGauges().containsKey(HoodieMetadataMetrics.INITIALIZE_STR + ".totalDuration"));
assertTrue((Long) metrics.getRegistry().getGauges().get(HoodieMetadataMetrics.INITIALIZE_STR + ".count").getValue() >= 1L);
@@ -3481,7 +3483,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
allRecords.addAll(secondBatchOfrecords);
// RI should have created mappings for all the records inserted above
- HoodieTableMetadata metadataReader = HoodieTableMetadata.create(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ HoodieTableMetadata metadataReader = HoodieTableMetadata.create(
+ context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
Map<String, List<HoodieRecordGlobalLocation>> result = metadataReader
.readRecordIndex(allRecords.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toList()));
assertEquals(allRecords.size(), (int) result.values().stream().flatMap(List::stream).count(), "RI should have mapping for all the records in firstCommit");
@@ -3496,7 +3499,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
client.delete(jsc.parallelize(recordsToDelete, 1).map(HoodieRecord::getKey), deleteTime);
// RI should not return mappings for deleted records
- metadataReader = HoodieTableMetadata.create(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ metadataReader = HoodieTableMetadata.create(
+ context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
result = metadataReader.readRecordIndex(allRecords.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toList()));
assertEquals(allRecords.size() - recordsToDelete.size(), result.size(), "RI should not have mapping for deleted records");
result.keySet().forEach(mappingKey -> assertFalse(keysToDelete.contains(mappingKey), "RI should not have mapping for deleted records"));
@@ -3509,11 +3513,12 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
String deleteTime = client.startCommit();
client.delete(jsc.emptyRDD(), deleteTime);
- HoodieTableMetadata metadataReader = HoodieTableMetadata.create(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ HoodieTableMetadata metadataReader = HoodieTableMetadata.create(
+ context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
assertTrue(metadataReader.getLatestCompactionTime().isPresent(), "Compaction should have taken place on MDT");
// RI should not return mappings for deleted records
- metadataReader = HoodieTableMetadata.create(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ metadataReader = HoodieTableMetadata.create(context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
Map<String, List<HoodieRecordGlobalLocation>> result = metadataReader.readRecordIndex(allRecords.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toList()));
assertEquals(allRecords.size() - keysToDelete.size(), result.size(), "RI should not have mapping for deleted records");
result.keySet().forEach(mappingKey -> assertFalse(keysToDelete.contains(mappingKey), "RI should not have mapping for deleted records"));
@@ -3523,7 +3528,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
client.upsert(jsc.parallelize(recordsToDelete, 1), reinsertTime).collect();
// New mappings should have been created for re-inserted records and should map to the new commit time
- metadataReader = HoodieTableMetadata.create(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ metadataReader = HoodieTableMetadata.create(context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
result = metadataReader.readRecordIndex(allRecords.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toList()));
assertEquals(allRecords.size(), (int) result.values().stream().flatMap(List::stream).count(), "RI should have mappings for re-inserted records");
for (String reInsertedKey : keysToDelete) {
@@ -3560,7 +3565,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// Partitions should match
FileSystemBackedTableMetadata fsBackedTableMetadata = new FileSystemBackedTableMetadata(
- engineContext, metaClient.getTableConfig(), storageConf, config.getBasePath());
+ engineContext, metaClient.getTableConfig(), metaClient.getStorage(), config.getBasePath());
List<String> fsPartitions = fsBackedTableMetadata.getAllPartitionPaths();
List<String> metadataPartitions = tableMetadata.getAllPartitionPaths();
@@ -3686,7 +3691,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// Metadata table has a fixed number of partitions
// Cannot use FSUtils.getAllFoldersWithPartitionMetaFile for this as that function filters all directory
// in the .hoodie folder.
- List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, getMetadataTableBasePath(basePath), false);
+ List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, storage, getMetadataTableBasePath(basePath), false);
assertEquals(metadataWriter.getEnabledPartitionTypes().size(), metadataTablePartitions.size());
final Map<String, MetadataPartitionType> metadataEnabledPartitionTypes = new HashMap<>();
@@ -3780,7 +3785,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
private HoodieTableMetadata metadata(SparkRDDWriteClient client) {
HoodieWriteConfig clientConfig = client.getConfig();
- return HoodieTableMetadata.create(client.getEngineContext(), clientConfig.getMetadataConfig(), clientConfig.getBasePath());
+ return HoodieTableMetadata.create(client.getEngineContext(), storage, clientConfig.getMetadataConfig(), clientConfig.getBasePath());
}
private void changeTableVersion(HoodieTableVersion version) throws IOException {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
index cec201ee754..a3c8783fece 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
@@ -124,7 +124,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
HoodieTableType tableType = HoodieTableType.COPY_ON_WRITE;
init(tableType);
testTable.doWriteOperation("000001", INSERT, emptyList(), asList("p1"), 1);
- HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), reuse);
+ HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), reuse);
assertTrue(tableMetadata.enabled());
List<String> metadataPartitions = tableMetadata.getAllPartitionPaths();
String partition = metadataPartitions.get(0);
@@ -168,7 +168,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
private void verifyBaseMetadataTable(boolean reuseMetadataReaders) throws IOException {
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(
- context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), reuseMetadataReaders);
+ context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), reuseMetadataReaders);
assertTrue(tableMetadata.enabled());
List<java.nio.file.Path> fsPartitionPaths = testTable.getAllPartitionPaths();
List<String> fsPartitions = new ArrayList<>();
@@ -211,7 +211,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
init(tableType);
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(context,
- writeConfig.getMetadataConfig(), writeConfig.getBasePath(), false);
+ storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), false);
assertEquals(HoodieTableMetadataKeyGenerator.class.getCanonicalName(),
tableMetadata.getMetadataMetaClient().getTableConfig().getKeyGeneratorClassName());
@@ -225,7 +225,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
public void testNotExistPartition(final HoodieTableType tableType) throws Exception {
init(tableType);
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(context,
- writeConfig.getMetadataConfig(), writeConfig.getBasePath(), false);
+ storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), false);
List<StoragePathInfo> allFilesInPartition = tableMetadata.getAllFilesInPartition(
new StoragePath(writeConfig.getBasePath() + "dummy"));
assertEquals(allFilesInPartition.size(), 0);
@@ -382,6 +382,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
private Set<String> getFilePathsInPartition(String partition) throws IOException {
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(
new HoodieLocalEngineContext(storageConf),
+ storage,
HoodieMetadataConfig.newBuilder().enable(true).build(),
basePath);
return tableMetadata.getAllFilesInPartition(new StoragePath(basePath, partition))
@@ -527,7 +528,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
final HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
- getHoodieSparkIOFactory(context.getStorageConf())
+ getHoodieSparkIOFactory(storage)
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(table.getConfig(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index e6becfec93a..335a0f8f86a 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -1215,7 +1215,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
String file1 = statuses.get(0).getFileId();
assertEquals(100,
- fileUtils.readRowKeys(storageConf, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
+ fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
.size(), "file should contain 100 records");
// Update + Inserts such that they just expand file1
@@ -1235,10 +1235,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(file1, statuses.get(0).getFileId(), "Existing file should be expanded");
assertEquals(commitTime1, statuses.get(0).getStat().getPrevCommit(), "Existing file should be expanded");
StoragePath newFile = new StoragePath(basePath, statuses.get(0).getStat().getPath());
- assertEquals(140, fileUtils.readRowKeys(storageConf, newFile).size(),
+ assertEquals(140, fileUtils.readRowKeys(storage, newFile).size(),
"file should contain 140 records");
- List<GenericRecord> records = fileUtils.readAvroRecords(storageConf, newFile);
+ List<GenericRecord> records = fileUtils.readAvroRecords(storage, newFile);
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
assertEquals(commitTime2, record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString(), "only expect commit2");
@@ -1269,7 +1269,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
for (HoodieBaseFile file : files) {
if (file.getFileName().contains(file1)) {
assertEquals(commitTime3, file.getCommitTime(), "Existing file should be expanded");
- records = fileUtils.readAvroRecords(storageConf, new StoragePath(file.getPath()));
+ records = fileUtils.readAvroRecords(storage, new StoragePath(file.getPath()));
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
String recordCommitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
@@ -1285,7 +1285,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(0, keys2.size(), "All keys added in commit 2 must be updated in commit3 correctly");
} else {
assertEquals(commitTime3, file.getCommitTime(), "New file must be written for commit 3");
- records = fileUtils.readAvroRecords(storageConf, new StoragePath(file.getPath()));
+ records = fileUtils.readAvroRecords(storage, new StoragePath(file.getPath()));
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
assertEquals(commitTime3, record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString(),
@@ -1326,7 +1326,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
String file1 = statuses.get(0).getFileId();
assertEquals(100,
- fileUtils.readRowKeys(storageConf, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
+ fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
.size(), "file should contain 100 records");
// Second, set of Inserts should just expand file1
@@ -1342,9 +1342,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(commitTime1, statuses.get(0).getStat().getPrevCommit(), "Existing file should be expanded");
StoragePath newFile = new StoragePath(basePath, statuses.get(0).getStat().getPath());
- assertEquals(140, fileUtils.readRowKeys(storageConf, newFile).size(),
+ assertEquals(140, fileUtils.readRowKeys(storage, newFile).size(),
"file should contain 140 records");
- List<GenericRecord> records = fileUtils.readAvroRecords(storageConf, newFile);
+ List<GenericRecord> records = fileUtils.readAvroRecords(storage, newFile);
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
String recCommitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
@@ -1363,8 +1363,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertNoWriteErrors(statuses);
assertEquals(2, statuses.size(), "2 files needs to be committed.");
assertEquals(340,
- fileUtils.readRowKeys(storageConf, new StoragePath(basePath, statuses.get(0).getStat().getPath())).size()
- + fileUtils.readRowKeys(storageConf, new StoragePath(basePath, statuses.get(1).getStat().getPath())).size(),
+ fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(0).getStat().getPath())).size()
+ + fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(1).getStat().getPath())).size(),
"file should contain 340 records");
HoodieTableMetaClient metaClient = createMetaClient(basePath);
@@ -1376,7 +1376,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
int totalInserts = 0;
for (HoodieBaseFile file : files) {
assertEquals(commitTime3, file.getCommitTime(), "All files must be at commit 3");
- totalInserts += fileUtils.readAvroRecords(storageConf, new StoragePath(file.getPath())).size();
+ totalInserts += fileUtils.readAvroRecords(storage, new StoragePath(file.getPath())).size();
}
assertEquals(totalInserts, inserts1.size() + inserts2.size() + inserts3.size(), "Total number of records must add up");
}
@@ -1410,7 +1410,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
String file1 = statuses.get(0).getFileId();
assertEquals(100, getFileUtilsInstance(metaClient).readRowKeys(
- storageConf, new StoragePath(basePath, statuses.get(0).getStat().getPath())).size(), "file should contain 100 records");
+ storage, new StoragePath(basePath, statuses.get(0).getStat().getPath())).size(), "file should contain 100 records");
// Delete 20 among 100 inserted
testDeletes(client, inserts1, 20, file1, "002", 80, keysSoFar);
@@ -2091,7 +2091,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
private Set<String> verifyRecordKeys(List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, List<GenericRecord> records) {
for (WriteStatus status : allStatus) {
StoragePath filePath = new StoragePath(basePath, status.getStat().getPath());
- records.addAll(getFileUtilsInstance(metaClient).readAvroRecords(storageConf, filePath));
+ records.addAll(getFileUtilsInstance(metaClient).readAvroRecords(storage, filePath));
}
Set<String> expectedKeys = recordsToRecordKeySet(expectedRecords);
assertEquals(records.size(), expectedKeys.size());
@@ -2180,10 +2180,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
StoragePath newFile = new StoragePath(basePath, statuses.get(0).getStat().getPath());
assertEquals(expectedRecords,
- getFileUtilsInstance(metaClient).readRowKeys(storageConf, newFile).size(),
+ getFileUtilsInstance(metaClient).readRowKeys(storage, newFile).size(),
"file should contain 110 records");
- List<GenericRecord> records = getFileUtilsInstance(metaClient).readAvroRecords(storageConf, newFile);
+ List<GenericRecord> records = getFileUtilsInstance(metaClient).readAvroRecords(storage, newFile);
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
assertTrue(keys.contains(recordKey), "key expected to be part of " + instantTime);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
index 1ac6a89c60b..1d9f58fcde7 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
@@ -120,7 +120,7 @@ public class TestRemoteFileSystemViewWithMetadataTable extends HoodieSparkClient
context, config.getViewStorageConfig(),
config.getCommonConfig(),
metaClient -> new HoodieBackedTestDelayedTableMetadata(
- context, config.getMetadataConfig(), metaClient.getBasePathV2().toString(), true)));
+ context, metaClient.getStorage(), config.getMetadataConfig(), metaClient.getBasePathV2().toString(), true)));
timelineService.startService();
timelineServicePort = timelineService.getServerPort();
LOG.info("Started timeline server on port: " + timelineServicePort);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java
index 3ecd9102548..496a922bb7d 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java
@@ -336,7 +336,7 @@ public class TestHoodieBloomIndex extends TestHoodieMetadataBase {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).build();
List<Pair<String, Long>> results = HoodieIndexUtils.filterKeysFromFile(
- new StoragePath(Paths.get(basePath, partition, filename).toString()), uuids, storageConf);
+ new StoragePath(Paths.get(basePath, partition, filename).toString()), uuids, storage);
assertEquals(results.size(), 2);
assertTrue(results.get(0).getLeft().equals("1eb5b87a-1feh-4edd-87b4-6ec96dc405a0")
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
index 5f85f4c9db3..400c9ac43c4 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
@@ -1000,7 +1000,7 @@ public class TestHoodieTimelineArchiver extends HoodieSparkClientTestHarness {
.build())
.withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA)
.withParallelism(2, 2).forTable("test-trip-table").build();
- HoodieMetrics metrics = new HoodieMetrics(cfg, storageConf);
+ HoodieMetrics metrics = new HoodieMetrics(cfg, storage);
BaseHoodieWriteClient client = getHoodieWriteClient(cfg);
client.archive();
assertTrue(metrics.getMetrics().getRegistry().getNames().contains(metrics.getMetricsName(ARCHIVE_ACTION, DURATION_STR)));
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java
index 74826c6f39b..f46fe9f6bcb 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java
@@ -55,7 +55,7 @@ public class TestHoodieAvroFileWriterFactory extends HoodieClientTestBase {
HoodieTable table = HoodieSparkTable.create(cfg, context, metaClient);
SparkTaskContextSupplier supplier = new SparkTaskContextSupplier();
HoodieFileWriter parquetWriter = HoodieFileWriterFactory.getFileWriter(instantTime,
- parquetPath, table.getStorageConf(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
+ parquetPath, table.getStorage(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
assertTrue(parquetWriter instanceof HoodieAvroParquetWriter);
parquetWriter.close();
@@ -63,7 +63,7 @@ public class TestHoodieAvroFileWriterFactory extends HoodieClientTestBase {
final StoragePath hfilePath = new StoragePath(
basePath + "/partition/path/f1_1-0-1_000.hfile");
HoodieFileWriter hfileWriter = HoodieFileWriterFactory.getFileWriter(instantTime,
- hfilePath, table.getStorageConf(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
+ hfilePath, table.getStorage(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
assertTrue(hfileWriter instanceof HoodieAvroHFileWriter);
hfileWriter.close();
@@ -71,7 +71,7 @@ public class TestHoodieAvroFileWriterFactory extends HoodieClientTestBase {
final StoragePath orcPath = new StoragePath(
basePath + "/partition/path/f1_1-0-1_000.orc");
HoodieFileWriter orcFileWriter = HoodieFileWriterFactory.getFileWriter(instantTime,
- orcPath, table.getStorageConf(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
+ orcPath, table.getStorage(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
assertTrue(orcFileWriter instanceof HoodieAvroOrcWriter);
orcFileWriter.close();
@@ -80,7 +80,7 @@ public class TestHoodieAvroFileWriterFactory extends HoodieClientTestBase {
basePath + "/partition/path/f.b51192a8-574b-4a85-b246-bcfec03ac8bf_100.log.2_1-0-1");
final Throwable thrown = assertThrows(UnsupportedOperationException.class, () -> {
HoodieFileWriterFactory.getFileWriter(instantTime, logPath,
- table.getStorageConf(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
+ table.getStorage(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
}, "should fail since log storage writer is not supported yet.");
assertTrue(thrown.getMessage().contains("format not supported yet."));
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
index 9b832bd6af6..af69b13c6be 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
@@ -206,14 +206,14 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestBase implemen
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
BloomFilter filter = FileFormatUtils.getInstance(table.getBaseFileFormat())
- .readBloomFilterFromMetadata(storageConf, new StoragePath(filePath.toUri()));
+ .readBloomFilterFromMetadata(storage, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
// Read the base file, check the record content
List<GenericRecord> fileRecords = FileFormatUtils.getInstance(table.getBaseFileFormat())
- .readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ .readAvroRecords(storage, new StoragePath(filePath.toUri()));
GenericRecord newRecord;
int index = 0;
for (GenericRecord record : fileRecords) {
@@ -248,7 +248,7 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestBase implemen
// Check whether the record has been updated
Path updatedFilePath = allFiles[0].getPath();
BloomFilter updatedFilter = getFileUtilsInstance(metaClient)
- .readBloomFilterFromMetadata(storageConf, new StoragePath(updatedFilePath.toUri()));
+ .readBloomFilterFromMetadata(storage, new StoragePath(updatedFilePath.toUri()));
for (HoodieRecord record : records) {
// No change to the _row_key
assertTrue(updatedFilter.mightContain(record.getRecordKey()));
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkers.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkers.java
index ec503a8774c..6c5a3d234c9 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkers.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkers.java
@@ -21,7 +21,6 @@ package org.apache.hudi.table.marker;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.util.CollectionUtils;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.testutils.HoodieClientTestUtils;
@@ -47,7 +46,7 @@ public class TestDirectWriteMarkers extends TestWriteMarkersBase {
this.jsc = new JavaSparkContext(
HoodieClientTestUtils.getSparkConfForTest(TestDirectWriteMarkers.class.getName()));
this.context = new HoodieSparkEngineContext(jsc);
- this.storage = HoodieStorageUtils.getStorage(metaClient.getBasePathV2(), metaClient.getStorageConf());
+ this.storage = metaClient.getStorage();
this.markerFolderPath = new StoragePath(Paths.get(metaClient.getMarkerFolderPath("000")).toUri());
this.writeMarkers = new DirectWriteMarkers(
storage, metaClient.getBasePathV2().toString(), markerFolderPath.toString(), "000");
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java
index 9d6cf92b99d..b01cc9af9ff 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java
@@ -28,7 +28,6 @@ import org.apache.hudi.common.table.view.FileSystemViewStorageType;
import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.MarkerUtils;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.testutils.HoodieClientTestUtils;
import org.apache.hudi.timeline.service.TimelineService;
@@ -59,7 +58,7 @@ public class TestTimelineServerBasedWriteMarkers extends TestWriteMarkersBase {
this.jsc = new JavaSparkContext(
HoodieClientTestUtils.getSparkConfForTest(TestTimelineServerBasedWriteMarkers.class.getName()));
this.context = new HoodieSparkEngineContext(jsc);
- this.storage = HoodieStorageUtils.getStorage(metaClient.getBasePathV2(), metaClient.getStorageConf());
+ this.storage = metaClient.getStorage();
this.markerFolderPath = new StoragePath(metaClient.getMarkerFolderPath("000"));
FileSystemViewStorageConfig storageConf =
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
index 3db773549c8..11011eabd68 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
@@ -207,7 +207,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
List<String> markerPaths = new ArrayList<>(writeMarkers.allMarkerFilePaths());
if (deletePartialMarkerFiles) {
String toDeleteMarkerFile = markerPaths.get(0);
- table.getMetaClient().getStorage().deleteDirectory(new StoragePath(
+ table.getStorage().deleteDirectory(new StoragePath(
table.getMetaClient().getTempFolderPath() + "/" + commitInstant.getTimestamp()
+ "/" + toDeleteMarkerFile));
markerPaths.remove(toDeleteMarkerFile);
@@ -621,7 +621,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
List<String> markerPaths = new ArrayList<>(writeMarkers.allMarkerFilePaths());
if (deletePartialMarkerFiles) {
String toDeleteMarkerFile = markerPaths.get(0);
- table.getMetaClient().getStorage().deleteDirectory(new StoragePath(
+ table.getStorage().deleteDirectory(new StoragePath(
table.getMetaClient().getTempFolderPath() + "/" + commitInstant.getTimestamp()
+ "/" + toDeleteMarkerFile));
markerPaths.remove(toDeleteMarkerFile);
@@ -660,7 +660,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
assertTrue(writeMarkers.doesMarkerDirExist());
assertEquals(0, getTimelineServerBasedMarkerFileCount(
table.getMetaClient().getMarkerFolderPath(commitInstant.getTimestamp()),
- (FileSystem) table.getMetaClient().getStorage().getFileSystem()));
+ (FileSystem) table.getStorage().getFileSystem()));
} else {
assertFalse(writeMarkers.doesMarkerDirExist());
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java
index 488b7e170d5..c95f71dc127 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java
@@ -29,7 +29,6 @@ import org.apache.hudi.common.testutils.minicluster.HdfsTestService;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -178,7 +177,7 @@ public class FunctionalTestHarness implements SparkProvider, DFSProvider, Hoodie
@AfterAll
public static synchronized void cleanUpAfterAll() throws IOException {
StoragePath workDir = new StoragePath("/tmp");
- HoodieStorage storage = HoodieStorageUtils.getStorage(
+ HoodieStorage storage = new HoodieHadoopStorage(
workDir, HadoopFSUtils.getStorageConf(hdfsTestService.getHadoopConf()));
List<StoragePathInfo> pathInfoList = storage.listDirectEntries(workDir);
for (StoragePathInfo f : pathInfoList) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
index 895040f1229..5b1f514982b 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
@@ -553,7 +553,7 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
public HoodieTableMetadata metadata(HoodieWriteConfig clientConfig,
HoodieEngineContext hoodieEngineContext) {
return HoodieTableMetadata.create(
- hoodieEngineContext, clientConfig.getMetadataConfig(), clientConfig.getBasePath());
+ hoodieEngineContext, storage, clientConfig.getMetadataConfig(), clientConfig.getBasePath());
}
protected void validateFilesPerPartition(HoodieTestTable testTable,
@@ -629,7 +629,8 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
// Metadata table has a fixed number of partitions
// Cannot use FSUtils.getAllFoldersWithPartitionMetaFile for this as that function filters all directory
// in the .hoodie folder.
- List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, HoodieTableMetadata.getMetadataTableBasePath(basePath), false);
+ List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(
+ engineContext, storage, HoodieTableMetadata.getMetadataTableBasePath(basePath), false);
List<MetadataPartitionType> enabledPartitionTypes = metadataWriter.getEnabledPartitionTypes();
diff --git a/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java b/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java
index 9a0eb0ec578..df431f438ef 100644
--- a/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java
+++ b/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java
@@ -42,6 +42,7 @@ import org.apache.hudi.expression.Expression;
import org.apache.hudi.internal.schema.Types;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -424,7 +425,7 @@ public abstract class BaseHoodieTableFileIndex implements AutoCloseable {
private void doRefresh() {
HoodieTimer timer = HoodieTimer.start();
- resetTableMetadata(createMetadataTable(engineContext, metadataConfig, basePath));
+ resetTableMetadata(createMetadataTable(engineContext, metaClient.getStorage(), metadataConfig, basePath));
// Make sure we reload active timeline
metaClient.reloadActiveTimeline();
@@ -503,10 +504,12 @@ public abstract class BaseHoodieTableFileIndex implements AutoCloseable {
private static HoodieTableMetadata createMetadataTable(
HoodieEngineContext engineContext,
+ HoodieStorage storage,
HoodieMetadataConfig metadataConfig,
StoragePath basePath
) {
- HoodieTableMetadata newTableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, basePath.toString(), true);
+ HoodieTableMetadata newTableMetadata = HoodieTableMetadata.create(
+ engineContext, storage, metadataConfig, basePath.toString(), true);
return newTableMetadata;
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
index 0309aee00a9..a595dcc17de 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
@@ -235,16 +235,22 @@ public class HoodieStorageConfig extends HoodieConfig {
+ "and it is loaded at runtime. This is only required when trying to "
+ "override the existing write context when `hoodie.datasource.write.row.writer.enable=true`.");
+ public static final ConfigProperty<String> HOODIE_STORAGE_CLASS = ConfigProperty
+ .key("hoodie.storage.class")
+ .defaultValue("org.apache.hudi.storage.hadoop.HoodieHadoopStorage")
+ .markAdvanced()
+ .sinceVersion("0.15.0")
+ .withDocumentation("The fully-qualified class name of the `HoodieStorage` implementation class to instantiate. "
+ + "The provided class should implement `org.apache.hudi.storage.HoodieStorage`");
+
public static final ConfigProperty<String> HOODIE_IO_FACTORY_CLASS = ConfigProperty
.key("hoodie.io.factory.class")
- .defaultValue("org.apache.hudi.io.storage.HoodieHadoopIOFactory")
+ .defaultValue("org.apache.hudi.io.hadoop.HoodieHadoopIOFactory")
.markAdvanced()
.sinceVersion("0.15.0")
.withDocumentation("The fully-qualified class name of the factory class to return readers and writers of files used "
+ "by Hudi. The provided class should implement `org.apache.hudi.io.storage.HoodieIOFactory`.");
-
-
/**
* @deprecated Use {@link #PARQUET_MAX_FILE_SIZE} and its methods instead
*/
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java
index 91038de1444..78f58db7eb5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieReaderContext.java
@@ -75,12 +75,12 @@ public abstract class HoodieReaderContext<T> {
* @param length Bytes to read.
* @param dataSchema Schema of records in the file in {@link Schema}.
* @param requiredSchema Schema containing required fields to read in {@link Schema} for projection.
- * @param conf {@link StorageConfiguration} for reading records.
+ * @param storage {@link HoodieStorage} for reading records.
* @return {@link ClosableIterator<T>} that can return all records through iteration.
*/
public abstract ClosableIterator<T> getFileRecordIterator(
StoragePath filePath, long start, long length, Schema dataSchema, Schema requiredSchema,
- StorageConfiguration<?> conf) throws IOException;
+ HoodieStorage storage) throws IOException;
/**
* Converts an Avro record, e.g., serialized in the log files, to an engine-specific record.
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
index 07230a9a69e..58924deb988 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
@@ -221,12 +221,14 @@ public class FSUtils {
}
}
- public static List<String> getAllPartitionPaths(HoodieEngineContext engineContext, String basePathStr,
+ public static List<String> getAllPartitionPaths(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ String basePathStr,
boolean useFileListingFromMetadata) {
HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder()
.enable(useFileListingFromMetadata)
.build();
- try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, basePathStr)) {
+ try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, storage, metadataConfig, basePathStr)) {
return tableMetadata.getAllPartitionPaths();
} catch (Exception e) {
throw new HoodieException("Error fetching partition paths from metadata table", e);
@@ -234,9 +236,10 @@ public class FSUtils {
}
public static List<String> getAllPartitionPaths(HoodieEngineContext engineContext,
+ HoodieStorage storage,
HoodieMetadataConfig metadataConfig,
String basePathStr) {
- try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig,
+ try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, storage, metadataConfig,
basePathStr)) {
return tableMetadata.getAllPartitionPaths();
} catch (Exception e) {
@@ -245,10 +248,11 @@ public class FSUtils {
}
public static Map<String, List<StoragePathInfo>> getFilesInPartitions(HoodieEngineContext engineContext,
+ HoodieStorage storage,
HoodieMetadataConfig metadataConfig,
String basePathStr,
String[] partitionPaths) {
- try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig,
+ try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, storage, metadataConfig,
basePathStr)) {
return tableMetadata.getAllFilesInPartitions(Arrays.asList(partitionPaths));
} catch (Exception ex) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java
index 31c39407ad7..b132daf97e5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java
@@ -23,8 +23,7 @@ import org.apache.hudi.common.util.JsonUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -174,11 +173,11 @@ public class HoodieCommitMetadata implements Serializable {
* been touched multiple times in the given commits, the return value will keep the one
* from the latest commit.
*
- * @param storageConf storage configuration.
+ * @param storage {@link HoodieStorage} instance.
* @param basePath The base path
* @return the file full path to file status mapping
*/
- public Map<String, StoragePathInfo> getFullPathToInfo(StorageConfiguration<?> storageConf,
+ public Map<String, StoragePathInfo> getFullPathToInfo(HoodieStorage storage,
String basePath) {
Map<String, StoragePathInfo> fullPathToInfoMap = new HashMap<>();
for (List<HoodieWriteStat> stats : getPartitionToWriteStats().values()) {
@@ -188,8 +187,7 @@ public class HoodieCommitMetadata implements Serializable {
StoragePath fullPath = relativeFilePath != null
? FSUtils.constructAbsolutePath(basePath, relativeFilePath) : null;
if (fullPath != null) {
- long blockSize =
- HoodieStorageUtils.getStorage(fullPath.toString(), storageConf).getDefaultBlockSize(fullPath);
+ long blockSize = storage.getDefaultBlockSize(fullPath);
StoragePathInfo pathInfo = new StoragePathInfo(
fullPath, stat.getFileSizeInBytes(), false, (short) 0, blockSize, 0);
fullPathToInfoMap.put(fullPath.getName(), pathInfo);
@@ -204,16 +202,14 @@ public class HoodieCommitMetadata implements Serializable {
* been touched multiple times in the given commits, the return value will keep the one
* from the latest commit by file group ID.
*
- * <p>Note: different with {@link #getFullPathToInfo(StorageConfiguration, String)},
+ * <p>Note: different with {@link #getFullPathToInfo(HoodieStorage, String)},
* only the latest commit file for a file group is returned,
* this is an optimization for COPY_ON_WRITE table to eliminate legacy files for filesystem view.
*
- * @param storageConf storage configuration.
* @param basePath The base path
* @return the file ID to file status mapping
*/
- public Map<String, StoragePathInfo> getFileIdToInfo(StorageConfiguration<?> storageConf,
- String basePath) {
+ public Map<String, StoragePathInfo> getFileIdToInfo(String basePath) {
Map<String, StoragePathInfo> fileIdToInfoMap = new HashMap<>();
for (List<HoodieWriteStat> stats : getPartitionToWriteStats().values()) {
// Iterate through all the written files.
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
index 5d75414c6ff..9256e6f4440 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
@@ -188,7 +188,7 @@ public class HoodiePartitionMetadata {
FileFormatUtils reader = FileFormatUtils.getInstance(metafilePath);
// Data file format
Map<String, String> metadata = reader.readFooter(
- storage.getConf(), true, metafilePath, PARTITION_DEPTH_KEY, COMMIT_TIME_KEY);
+ storage, true, metafilePath, PARTITION_DEPTH_KEY, COMMIT_TIME_KEY);
props.clear();
props.putAll(metadata);
format = Option.of(reader.getFormat());
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
index 62a79f2b080..d6f815ae8fd 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -384,13 +384,12 @@ public class HoodieTableMetaClient implements Serializable {
public HoodieStorage getStorage() {
if (storage == null) {
+ HoodieStorage newStorage = HoodieStorageUtils.getStorage(metaPath, getStorageConf());
ConsistencyGuard consistencyGuard = consistencyGuardConfig.isConsistencyCheckEnabled()
- ? new FailSafeConsistencyGuard(
- HoodieStorageUtils.getStorage(metaPath, getStorageConf()),
- consistencyGuardConfig)
+ ? new FailSafeConsistencyGuard(newStorage, consistencyGuardConfig)
: new NoOpConsistencyGuard();
- storage = getIOFactory(getStorageConf()).getStorage(metaPath,
+ storage = getIOFactory(newStorage).getStorage(metaPath,
fileSystemRetryConfig.isFileSystemActionRetryEnable(),
fileSystemRetryConfig.getMaxRetryIntervalMs(),
fileSystemRetryConfig.getMaxRetryNumbers(),
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
index aec5dc73ad6..5c5d6133e96 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
@@ -306,7 +306,7 @@ public class TableSchemaResolver {
.orElseThrow(() -> new IllegalArgumentException("Could not find any data file written for compaction "
+ lastCompactionCommit + ", could not get schema for table " + metaClient.getBasePath()));
StoragePath path = new StoragePath(filePath);
- return FileFormatUtils.getInstance(path).readAvroSchema(metaClient.getStorageConf(), path);
+ return FileFormatUtils.getInstance(path).readAvroSchema(metaClient.getStorage(), path);
}
private Schema readSchemaFromLogFile(StoragePath path) throws IOException {
@@ -473,7 +473,7 @@ public class TableSchemaResolver {
// this is a log file
schema = readSchemaFromLogFile(filePath);
} else {
- schema = FileFormatUtils.getInstance(filePath).readAvroSchema(metaClient.getStorageConf(), filePath);
+ schema = FileFormatUtils.getInstance(filePath).readAvroSchema(metaClient.getStorage(), filePath);
}
}
return schema;
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java
index 8b24367bd8a..f2399d6b080 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java
@@ -40,7 +40,6 @@ import org.apache.hudi.internal.schema.InternalSchema;
import org.apache.hudi.io.SeekableDataInputStream;
import org.apache.hudi.io.util.IOUtils;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StorageSchemes;
@@ -72,7 +71,6 @@ public class HoodieLogFileReader implements HoodieLogFormat.Reader {
private static final String REVERSE_LOG_READER_HAS_NOT_BEEN_ENABLED = "Reverse log reader has not been enabled";
private final HoodieStorage storage;
- private final StorageConfiguration<?> storageConf;
private final HoodieLogFile logFile;
private final int bufferSize;
private final byte[] magicBuffer = new byte[6];
@@ -103,7 +101,6 @@ public class HoodieLogFileReader implements HoodieLogFormat.Reader {
public HoodieLogFileReader(HoodieStorage storage, HoodieLogFile logFile, Schema readerSchema, int bufferSize, boolean reverseReader,
boolean enableRecordLookups, String keyField, InternalSchema internalSchema) throws IOException {
this.storage = storage;
- this.storageConf = this.storage.getConf();
// NOTE: We repackage {@code HoodieLogFile} here to make sure that the provided path
// is prefixed with an appropriate scheme given that we're not propagating the FS
// further
@@ -184,7 +181,7 @@ public class HoodieLogFileReader implements HoodieLogFormat.Reader {
long blockEndPos = inputStream.getPos();
HoodieLogBlock.HoodieLogBlockContentLocation logBlockContentLoc =
- new HoodieLogBlock.HoodieLogBlockContentLocation(storageConf, logFile, contentPosition, contentLength, blockEndPos);
+ new HoodieLogBlock.HoodieLogBlockContentLocation(storage, logFile, contentPosition, contentLength, blockEndPos);
switch (Objects.requireNonNull(blockType)) {
case AVRO_DATA_BLOCK:
@@ -259,7 +256,7 @@ public class HoodieLogFileReader implements HoodieLogFormat.Reader {
long contentPosition = inputStream.getPos();
Option<byte[]> corruptedBytes = HoodieLogBlock.tryReadContent(inputStream, corruptedBlockSize, true);
HoodieLogBlock.HoodieLogBlockContentLocation logBlockContentLoc =
- new HoodieLogBlock.HoodieLogBlockContentLocation(storageConf, logFile, contentPosition, corruptedBlockSize, nextBlockOffset);
+ new HoodieLogBlock.HoodieLogBlockContentLocation(storage, logFile, contentPosition, corruptedBlockSize, nextBlockOffset);
return new HoodieCorruptBlock(corruptedBytes, () -> getDataInputStream(storage, this.logFile, bufferSize), true, Option.of(logBlockContentLoc), new HashMap<>(), new HashMap<>());
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java
index 8247273aab0..2ec1beb5e74 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java
@@ -30,7 +30,7 @@ import org.apache.hudi.common.util.collection.CloseableMappingIterator;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.internal.schema.InternalSchema;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericDatumReader;
@@ -101,7 +101,7 @@ public class HoodieAvroDataBlock extends HoodieDataBlock {
}
@Override
- protected byte[] serializeRecords(List<HoodieRecord> records, StorageConfiguration<?> storageConf) throws IOException {
+ protected byte[] serializeRecords(List<HoodieRecord> records, HoodieStorage storage) throws IOException {
Schema schema = new Schema.Parser().parse(super.getLogBlockHeader().get(HeaderMetadataType.SCHEMA));
GenericDatumWriter<IndexedRecord> writer = new GenericDatumWriter<>(schema);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java
index a519f80eb40..ac0d02e42c2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java
@@ -20,7 +20,7 @@ package org.apache.hudi.common.table.log.block;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import java.util.HashMap;
import java.util.Map;
@@ -62,7 +62,7 @@ public class HoodieCommandBlock extends HoodieLogBlock {
}
@Override
- public byte[] getContentBytes(StorageConfiguration<?> storageConf) {
+ public byte[] getContentBytes(HoodieStorage storage) {
return new byte[0];
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java
index 74502ee1b8b..4de71e7759e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java
@@ -20,7 +20,7 @@ package org.apache.hudi.common.table.log.block;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import java.io.IOException;
import java.util.Map;
@@ -39,7 +39,7 @@ public class HoodieCorruptBlock extends HoodieLogBlock {
}
@Override
- public byte[] getContentBytes(StorageConfiguration<?> storageConf) throws IOException {
+ public byte[] getContentBytes(HoodieStorage storage) throws IOException {
if (!getContent().isPresent() && readBlockLazily) {
// read content from disk
inflate();
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java
index 0819a5f8e53..1a0cdac44b9 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java
@@ -25,7 +25,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.avro.Schema;
import org.slf4j.Logger;
@@ -135,7 +135,7 @@ public abstract class HoodieDataBlock extends HoodieLogBlock {
}
@Override
- public byte[] getContentBytes(StorageConfiguration<?> storageConf) throws IOException {
+ public byte[] getContentBytes(HoodieStorage storage) throws IOException {
// In case this method is called before realizing records from content
Option<byte[]> content = getContent();
@@ -145,7 +145,7 @@ public abstract class HoodieDataBlock extends HoodieLogBlock {
return content.get();
}
- return serializeRecords(records.get(), storageConf);
+ return serializeRecords(records.get(), storage);
}
public String getKeyFieldName() {
@@ -286,7 +286,7 @@ public abstract class HoodieDataBlock extends HoodieLogBlock {
);
}
- protected abstract byte[] serializeRecords(List<HoodieRecord> records, StorageConfiguration<?> storageConf) throws IOException;
+ protected abstract byte[] serializeRecords(List<HoodieRecord> records, HoodieStorage storage) throws IOException;
protected abstract <T> ClosableIterator<HoodieRecord<T>> deserializeRecords(byte[] content, HoodieRecordType type) throws IOException;
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java
index 835fed9d44b..bb692312f6a 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java
@@ -28,7 +28,7 @@ import org.apache.hudi.common.util.SerializationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.util.Lazy;
import org.apache.avro.io.BinaryDecoder;
@@ -113,7 +113,7 @@ public class HoodieDeleteBlock extends HoodieLogBlock {
}
@Override
- public byte[] getContentBytes(StorageConfiguration<?> storageConf) throws IOException {
+ public byte[] getContentBytes(HoodieStorage storage) throws IOException {
Option<byte[]> content = getContent();
// In case this method is called before realizing keys from content
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java
index 8df2ee4e6a6..db42a93a274 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java
@@ -34,7 +34,6 @@ import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.inline.InLineFSUtils;
@@ -103,11 +102,11 @@ public class HoodieHFileDataBlock extends HoodieDataBlock {
}
@Override
- protected byte[] serializeRecords(List<HoodieRecord> records, StorageConfiguration<?> storageConf) throws IOException {
+ protected byte[] serializeRecords(List<HoodieRecord> records, HoodieStorage storage) throws IOException {
Schema writerSchema = new Schema.Parser().parse(
super.getLogBlockHeader().get(HoodieLogBlock.HeaderMetadataType.SCHEMA));
return FileFormatUtils.getInstance(HoodieFileFormat.HFILE).serializeRecordsToLogBlock(
- storageConf, records, writerSchema, getSchema(), getKeyFieldName(),
+ storage, records, writerSchema, getSchema(), getKeyFieldName(),
Collections.singletonMap(HFILE_COMPRESSION_ALGORITHM_NAME.key(), compressionCodec.get()));
}
@@ -115,13 +114,13 @@ public class HoodieHFileDataBlock extends HoodieDataBlock {
protected <T> ClosableIterator<HoodieRecord<T>> deserializeRecords(byte[] content, HoodieRecordType type) throws IOException {
checkState(readerSchema != null, "Reader's schema has to be non-null");
- StorageConfiguration<?> storageConf = getBlockContentLocation().get().getStorageConf().getInline();
- HoodieStorage storage = HoodieStorageUtils.getStorage(pathForReader, storageConf);
+ StorageConfiguration<?> storageConf = getBlockContentLocation().get().getStorage().getConf().getInline();
+ HoodieStorage inlineStorage = getBlockContentLocation().get().getStorage().newInstance(pathForReader, storageConf);
// Read the content
- try (HoodieFileReader reader = HoodieIOFactory.getIOFactory(storageConf)
+ try (HoodieFileReader reader = HoodieIOFactory.getIOFactory(inlineStorage)
.getReaderFactory(HoodieRecordType.AVRO)
.getContentReader(hFileReaderConfig, pathForReader, HoodieFileFormat.HFILE,
- storage, content, Option.of(getSchemaFromHeader()))) {
+ inlineStorage, content, Option.of(getSchemaFromHeader()))) {
return unsafeCast(reader.getRecordIterator(readerSchema));
}
}
@@ -130,12 +129,12 @@ public class HoodieHFileDataBlock extends HoodieDataBlock {
protected <T> ClosableIterator<T> deserializeRecords(HoodieReaderContext<T> readerContext, byte[] content) throws IOException {
checkState(readerSchema != null, "Reader's schema has to be non-null");
- StorageConfiguration<?> storageConf = getBlockContentLocation().get().getStorageConf().getInline();
- HoodieStorage storage = HoodieStorageUtils.getStorage(pathForReader, storageConf);
+ StorageConfiguration<?> storageConf = getBlockContentLocation().get().getStorage().getConf().getInline();
+ HoodieStorage inlineStorage = getBlockContentLocation().get().getStorage().newInstance(pathForReader, storageConf);
// Read the content
try (HoodieAvroHFileReaderImplBase reader = (HoodieAvroHFileReaderImplBase)
- HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecordType.AVRO).getContentReader(
- hFileReaderConfig, pathForReader, HoodieFileFormat.HFILE, storage, content,
+ HoodieIOFactory.getIOFactory(inlineStorage).getReaderFactory(HoodieRecordType.AVRO).getContentReader(
+ hFileReaderConfig, pathForReader, HoodieFileFormat.HFILE, inlineStorage, content,
Option.of(getSchemaFromHeader()))) {
return unsafeCast(reader.getIndexedRecordIterator(readerSchema, readerSchema));
}
@@ -148,15 +147,16 @@ public class HoodieHFileDataBlock extends HoodieDataBlock {
// NOTE: It's important to extend Hadoop configuration here to make sure configuration
// is appropriately carried over
- StorageConfiguration<?> inlineConf = blockContentLoc.getStorageConf().getInline();
-
+ StorageConfiguration<?> inlineConf = getBlockContentLocation().get().getStorage().getConf().getInline();
StoragePath inlinePath = InLineFSUtils.getInlineFilePath(
blockContentLoc.getLogFile().getPath(),
blockContentLoc.getLogFile().getPath().toUri().getScheme(),
blockContentLoc.getContentPositionInLogFile(),
blockContentLoc.getBlockSize());
+ HoodieStorage inlineStorage = getBlockContentLocation().get().getStorage().newInstance(inlinePath, inlineConf);
- try (final HoodieAvroHFileReaderImplBase reader = (HoodieAvroHFileReaderImplBase) HoodieIOFactory.getIOFactory(inlineConf)
+ try (final HoodieAvroHFileReaderImplBase reader = (HoodieAvroHFileReaderImplBase) HoodieIOFactory
+ .getIOFactory(inlineStorage)
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(hFileReaderConfig, inlinePath, HoodieFileFormat.HFILE, Option.of(getSchemaFromHeader()))) {
// Get writer's schema from the header
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java
index f04b7735997..33015e0c64f 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java
@@ -26,7 +26,7 @@ import org.apache.hudi.common.util.TypeUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.roaringbitmap.longlong.Roaring64NavigableMap;
import org.slf4j.Logger;
@@ -88,7 +88,7 @@ public abstract class HoodieLogBlock {
}
// Return the bytes representation of the data belonging to a LogBlock
- public byte[] getContentBytes(StorageConfiguration<?> storageConf) throws IOException {
+ public byte[] getContentBytes(HoodieStorage storage) throws IOException {
throw new HoodieException("No implementation was provided");
}
@@ -200,7 +200,7 @@ public abstract class HoodieLogBlock {
*/
public static final class HoodieLogBlockContentLocation {
// Storage Config required to access the file
- private final StorageConfiguration<?> storageConf;
+ private final HoodieStorage storage;
// The logFile that contains this block
private final HoodieLogFile logFile;
// The filePosition in the logFile for the contents of this block
@@ -210,20 +210,20 @@ public abstract class HoodieLogBlock {
// The final position where the complete block ends
private final long blockEndPos;
- public HoodieLogBlockContentLocation(StorageConfiguration<?> storageConf,
+ public HoodieLogBlockContentLocation(HoodieStorage storage,
HoodieLogFile logFile,
long contentPositionInLogFile,
long blockSize,
long blockEndPos) {
- this.storageConf = storageConf;
+ this.storage = storage;
this.logFile = logFile;
this.contentPositionInLogFile = contentPositionInLogFile;
this.blockSize = blockSize;
this.blockEndPos = blockEndPos;
}
- public StorageConfiguration<?> getStorageConf() {
- return storageConf;
+ public HoodieStorage getStorage() {
+ return storage;
}
public HoodieLogFile getLogFile() {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java
index 616177f80b6..6c17d9abf49 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java
@@ -26,6 +26,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.io.SeekableDataInputStream;
import org.apache.hudi.io.storage.HoodieIOFactory;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.inline.InLineFSUtils;
@@ -89,7 +90,7 @@ public class HoodieParquetDataBlock extends HoodieDataBlock {
}
@Override
- protected byte[] serializeRecords(List<HoodieRecord> records, StorageConfiguration<?> storageConf) throws IOException {
+ protected byte[] serializeRecords(List<HoodieRecord> records, HoodieStorage storage) throws IOException {
Map<String, String> paramsMap = new HashMap<>();
paramsMap.put(PARQUET_COMPRESSION_CODEC_NAME.key(), compressionCodecName.get());
paramsMap.put(PARQUET_COMPRESSION_RATIO_FRACTION.key(), String.valueOf(expectedCompressionRatio.get()));
@@ -98,7 +99,7 @@ public class HoodieParquetDataBlock extends HoodieDataBlock {
super.getLogBlockHeader().get(HoodieLogBlock.HeaderMetadataType.SCHEMA));
return FileFormatUtils.getInstance(PARQUET).serializeRecordsToLogBlock(
- storageConf, records, writerSchema, getSchema(), getKeyFieldName(), paramsMap);
+ storage, records, writerSchema, getSchema(), getKeyFieldName(), paramsMap);
}
/**
@@ -112,17 +113,18 @@ public class HoodieParquetDataBlock extends HoodieDataBlock {
// NOTE: It's important to extend Hadoop configuration here to make sure configuration
// is appropriately carried over
- StorageConfiguration<?> inlineConf = blockContentLoc.getStorageConf().getInline();
-
+ StorageConfiguration<?> inlineConf = blockContentLoc.getStorage().getConf().getInline();
StoragePath inlineLogFilePath = InLineFSUtils.getInlineFilePath(
blockContentLoc.getLogFile().getPath(),
blockContentLoc.getLogFile().getPath().toUri().getScheme(),
blockContentLoc.getContentPositionInLogFile(),
blockContentLoc.getBlockSize());
+ HoodieStorage inlineStorage = getBlockContentLocation().get().getStorage().newInstance(inlineLogFilePath, inlineConf);
Schema writerSchema = new Schema.Parser().parse(this.getLogBlockHeader().get(HeaderMetadataType.SCHEMA));
- ClosableIterator<HoodieRecord<T>> iterator = HoodieIOFactory.getIOFactory(inlineConf).getReaderFactory(type)
+ ClosableIterator<HoodieRecord<T>> iterator = HoodieIOFactory.getIOFactory(inlineStorage)
+ .getReaderFactory(type)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, inlineLogFilePath, PARQUET, Option.empty())
.getRecordIterator(writerSchema, readerSchema);
return iterator;
@@ -134,13 +136,14 @@ public class HoodieParquetDataBlock extends HoodieDataBlock {
// NOTE: It's important to extend Hadoop configuration here to make sure configuration
// is appropriately carried over
- StorageConfiguration<?> inlineConf = blockContentLoc.getStorageConf().getInline();
+ StorageConfiguration<?> inlineConf = blockContentLoc.getStorage().getConf().getInline();
StoragePath inlineLogFilePath = InLineFSUtils.getInlineFilePath(
blockContentLoc.getLogFile().getPath(),
blockContentLoc.getLogFile().getPath().toUri().getScheme(),
blockContentLoc.getContentPositionInLogFile(),
blockContentLoc.getBlockSize());
+ HoodieStorage inlineStorage = blockContentLoc.getStorage().newInstance(inlineLogFilePath, inlineConf);
Schema writerSchema =
new Schema.Parser().parse(this.getLogBlockHeader().get(HeaderMetadataType.SCHEMA));
@@ -149,7 +152,7 @@ public class HoodieParquetDataBlock extends HoodieDataBlock {
inlineLogFilePath, 0, blockContentLoc.getBlockSize(),
writerSchema,
readerSchema,
- inlineConf);
+ inlineStorage);
}
@Override
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java
index 22600bc7c95..0f9e9330914 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java
@@ -38,7 +38,6 @@ import org.apache.hudi.common.util.collection.ExternalSpillableMap;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -71,7 +70,7 @@ public final class HoodieFileGroupReader<T> implements Closeable {
private final HoodieReaderContext<T> readerContext;
private final Option<HoodieBaseFile> hoodieBaseFileOption;
private final List<HoodieLogFile> logFiles;
- private final StorageConfiguration<?> storageConf;
+ private final HoodieStorage storage;
private final TypedProperties props;
// Byte offset to start reading from the base file
private final long start;
@@ -96,7 +95,7 @@ public final class HoodieFileGroupReader<T> implements Closeable {
private final Option<UnaryOperator<T>> outputConverter;
public HoodieFileGroupReader(HoodieReaderContext<T> readerContext,
- StorageConfiguration<?> storageConf,
+ HoodieStorage storage,
String tablePath,
String latestCommitTime,
FileSlice fileSlice,
@@ -112,7 +111,7 @@ public final class HoodieFileGroupReader<T> implements Closeable {
ExternalSpillableMap.DiskMapType diskMapType,
boolean isBitCaskDiskMapCompressionEnabled) {
this.readerContext = readerContext;
- this.storageConf = storageConf;
+ this.storage = storage;
this.hoodieBaseFileOption = fileSlice.getBaseFile();
this.logFiles = fileSlice.getLogFiles().sorted(HoodieLogFile.getLogFileComparator()).collect(Collectors.toList());
this.props = props;
@@ -171,7 +170,7 @@ public final class HoodieFileGroupReader<T> implements Closeable {
return readerContext.getFileRecordIterator(
baseFile.getStoragePath(), start,
length,
- dataSchema, requiredSchema, storageConf);
+ dataSchema, requiredSchema, storage);
}
private Schema generateRequiredSchema() {
@@ -240,7 +239,7 @@ public final class HoodieFileGroupReader<T> implements Closeable {
dataFile.getStoragePath(), 0,
dataFile.getFileLen(),
createSchemaFromFields(allFields.getRight()),
- createSchemaFromFields(requiredFields.getRight()), storageConf));
+ createSchemaFromFields(requiredFields.getRight()), storage));
Option<ClosableIterator<T>> skeletonFileIterator =
requiredFields.getLeft().isEmpty() ? Option.empty() :
@@ -248,7 +247,7 @@ public final class HoodieFileGroupReader<T> implements Closeable {
baseFile.getStoragePath(), 0,
baseFile.getFileLen(),
createSchemaFromFields(allFields.getLeft()),
- createSchemaFromFields(requiredFields.getLeft()), storageConf));
+ createSchemaFromFields(requiredFields.getLeft()), storage));
if (!dataFileIterator.isPresent() && !skeletonFileIterator.isPresent()) {
throw new IllegalStateException("should not be here if only partition cols are required");
} else if (!dataFileIterator.isPresent()) {
@@ -286,8 +285,6 @@ public final class HoodieFileGroupReader<T> implements Closeable {
private void scanLogFiles() {
String path = readerState.tablePath;
- HoodieStorage storage = readerContext.getStorage(path, storageConf);
-
HoodieMergedLogRecordReader logRecordReader = HoodieMergedLogRecordReader.newBuilder()
.withHoodieReaderContext(readerContext)
.withStorage(storage)
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
index 376b6535af9..2866a7ab067 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
@@ -30,7 +30,6 @@ import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
import org.slf4j.Logger;
@@ -919,10 +918,9 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
StoragePath srcPath = new StoragePath(metaClient.getMetaPath(), getInstantFileName(instant));
StoragePath dstPath = new StoragePath(dstDir, getInstantFileName(instant));
try {
- HoodieStorage srcStorage = HoodieStorageUtils.getStorage(srcPath, metaClient.getStorageConf());
- HoodieStorage dstStorage = HoodieStorageUtils.getStorage(dstPath, metaClient.getStorageConf());
- dstStorage.createDirectory(dstDir);
- FileIOUtils.copy(srcStorage, srcPath, dstStorage, dstPath, false, true);
+ HoodieStorage storage = metaClient.getStorage();
+ storage.createDirectory(dstDir);
+ FileIOUtils.copy(storage, srcPath, storage, dstPath, false, true);
} catch (IOException e) {
throw new HoodieIOException("Could not copy instant from " + srcPath + " to " + dstPath, e);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
index bffaf0c88fc..1e6050d60f6 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
@@ -266,7 +266,7 @@ public class HoodieArchivedTimeline extends HoodieDefaultTimeline {
.filter(fileName -> filter == null || LSMTimeline.isFileInRange(filter, fileName))
.parallel().forEach(fileName -> {
// Read the archived file
- try (HoodieAvroFileReader reader = (HoodieAvroFileReader) HoodieIOFactory.getIOFactory(metaClient.getStorageConf())
+ try (HoodieAvroFileReader reader = (HoodieAvroFileReader) HoodieIOFactory.getIOFactory(metaClient.getStorage())
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, new StoragePath(metaClient.getArchivePath(), fileName))) {
try (ClosableIterator<IndexedRecord> iterator = reader.getIndexedRecordIterator(HoodieLSMTimelineInstant.getClassSchema(), readSchema)) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
index 30eefc92907..d34952aa0c8 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
@@ -220,7 +220,7 @@ public class FileSystemViewManager {
final FileSystemViewStorageConfig config,
final HoodieCommonConfig commonConfig) {
return createViewManager(context, config, commonConfig,
- metaClient -> HoodieTableMetadata.create(context, metadataConfig, metaClient.getBasePathV2().toString(), true));
+ metaClient -> HoodieTableMetadata.create(context, metaClient.getStorage(), metadataConfig, metaClient.getBasePathV2().toString(), true));
}
public static FileSystemViewManager createViewManager(final HoodieEngineContext context,
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java
index 44655a57fb9..468a0df68ee 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java
@@ -31,7 +31,6 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -131,25 +130,25 @@ public abstract class FileFormatUtils {
/**
* Read the rowKey list from the given data file.
*
- * @param configuration configuration to build storage object.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return set of row keys
*/
- public Set<String> readRowKeys(StorageConfiguration<?> configuration, StoragePath filePath) {
- return filterRowKeys(configuration, filePath, new HashSet<>())
+ public Set<String> readRowKeys(HoodieStorage storage, StoragePath filePath) {
+ return filterRowKeys(storage, filePath, new HashSet<>())
.stream().map(Pair::getKey).collect(Collectors.toSet());
}
/**
* Read the bloom filter from the metadata of the given data file.
*
- * @param configuration configuration.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return a BloomFilter object.
*/
- public BloomFilter readBloomFilterFromMetadata(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public BloomFilter readBloomFilterFromMetadata(HoodieStorage storage, StoragePath filePath) {
Map<String, String> footerVals =
- readFooter(configuration, false, filePath,
+ readFooter(storage, false, filePath,
HoodieBloomFilterWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY,
HoodieBloomFilterWriteSupport.OLD_HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY,
HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE);
@@ -173,12 +172,12 @@ public abstract class FileFormatUtils {
/**
* Read the min and max record key from the metadata of the given data file.
*
- * @param configuration configuration.
- * @param filePath the data file path.
- * @return a array of two string where the first is min record key and the second is max record key.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
+ * @return an array of two string where the first is min record key and the second is max record key.
*/
- public String[] readMinMaxRecordKeys(StorageConfiguration<?> configuration, StoragePath filePath) {
- Map<String, String> minMaxKeys = readFooter(configuration, true, filePath,
+ public String[] readMinMaxRecordKeys(HoodieStorage storage, StoragePath filePath) {
+ Map<String, String> minMaxKeys = readFooter(storage, true, filePath,
HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER, HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER);
if (minMaxKeys.size() != 2) {
throw new HoodieException(
@@ -193,114 +192,114 @@ public abstract class FileFormatUtils {
* Read the data file
* NOTE: This literally reads the entire file contents, thus should be used with caution.
*
- * @param configuration configuration.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return a list of GenericRecord.
*/
- public abstract List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath);
+ public abstract List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath);
/**
* Read the data file using the given schema
* NOTE: This literally reads the entire file contents, thus should be used with caution.
*
- * @param configuration configuration.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return a list of GenericRecord.
*/
- public abstract List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath, Schema schema);
+ public abstract List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath, Schema schema);
/**
* Read the footer data of the given data file.
*
- * @param configuration configuration.
- * @param required require the footer data to be in data file.
- * @param filePath the data file path.
- * @param footerNames the footer names to read.
+ * @param storage {@link HoodieStorage} instance.
+ * @param required require the footer data to be in data file.
+ * @param filePath the data file path.
+ * @param footerNames the footer names to read.
* @return a map where the key is the footer name and the value is the footer value.
*/
- public abstract Map<String, String> readFooter(StorageConfiguration<?> configuration, boolean required, StoragePath filePath,
+ public abstract Map<String, String> readFooter(HoodieStorage storage, boolean required, StoragePath filePath,
String... footerNames);
/**
* Returns the number of records in the data file.
*
- * @param configuration configuration.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
*/
- public abstract long getRowCount(StorageConfiguration<?> configuration, StoragePath filePath);
+ public abstract long getRowCount(HoodieStorage storage, StoragePath filePath);
/**
* Read the rowKey list matching the given filter, from the given data file.
* If the filter is empty, then this will return all the row keys and corresponding positions.
*
- * @param configuration configuration to build storage object.
- * @param filePath the data file path.
- * @param filter record keys filter.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
+ * @param filter record keys filter.
* @return set of pairs of row key and position matching candidateRecordKeys.
*/
- public abstract Set<Pair<String, Long>> filterRowKeys(StorageConfiguration<?> configuration, StoragePath filePath, Set<String> filter);
+ public abstract Set<Pair<String, Long>> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set<String> filter);
/**
* Fetch {@link HoodieKey}s with positions from the given data file.
*
- * @param configuration configuration to build storage object.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return {@link List} of pairs of {@link HoodieKey} and position fetched from the data file.
*/
- public abstract List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(StorageConfiguration<?> configuration, StoragePath filePath);
+ public abstract List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath);
/**
* Provides a closable iterator for reading the given data file.
*
- * @param configuration configuration to build storage object.
+ * @param storage {@link HoodieStorage} instance.
* @param filePath the data file path.
* @param keyGeneratorOpt instance of KeyGenerator.
* @return {@link ClosableIterator} of {@link HoodieKey}s for reading the file.
*/
- public abstract ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration,
+ public abstract ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage,
StoragePath filePath,
Option<BaseKeyGenerator> keyGeneratorOpt);
/**
* Provides a closable iterator for reading the given data file.
*
- * @param configuration configuration to build storage object.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return {@link ClosableIterator} of {@link HoodieKey}s for reading the file.
*/
- public abstract ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath);
+ public abstract ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath);
/**
* Fetch {@link HoodieKey}s with positions from the given data file.
*
- * @param configuration configuration to build storage object.
+ * @param storage {@link HoodieStorage} instance.
* @param filePath the data file path.
* @param keyGeneratorOpt instance of KeyGenerator.
* @return {@link List} of pairs of {@link HoodieKey} and position fetched from the data file.
*/
- public abstract List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(StorageConfiguration<?> configuration,
+ public abstract List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieStorage storage,
StoragePath filePath,
Option<BaseKeyGenerator> keyGeneratorOpt);
/**
* Read the Avro schema of the data file.
*
- * @param configuration configuration.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return the Avro schema of the data file.
*/
- public abstract Schema readAvroSchema(StorageConfiguration<?> configuration, StoragePath filePath);
+ public abstract Schema readAvroSchema(HoodieStorage storage, StoragePath filePath);
/**
* Reads column statistics stored in the metadata.
*
- * @param storageConf storage configuration.
- * @param filePath the data file path.
- * @param columnList List of columns to get column statistics.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
+ * @param columnList List of columns to get column statistics.
* @return {@link List} of {@link HoodieColumnRangeMetadata}.
*/
@SuppressWarnings("rawtype")
- public abstract List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(StorageConfiguration<?> storageConf,
+ public abstract List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(HoodieStorage storage,
StoragePath filePath,
List<String> columnList);
@@ -324,7 +323,7 @@ public abstract class FileFormatUtils {
/**
* Serializes Hudi records to the log block.
*
- * @param storageConf storage configuration.
+ * @param storage {@link HoodieStorage} instance.
* @param records a list of {@link HoodieRecord}.
* @param writerSchema writer schema string from the log block header.
* @param readerSchema
@@ -333,7 +332,7 @@ public abstract class FileFormatUtils {
* @return byte array after serialization.
* @throws IOException upon serialization error.
*/
- public abstract byte[] serializeRecordsToLogBlock(StorageConfiguration<?> storageConf,
+ public abstract byte[] serializeRecordsToLogBlock(HoodieStorage storage,
List<HoodieRecord> records,
Schema writerSchema,
Schema readerSchema, String keyFieldName,
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java b/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java
index 407cd7103e3..8e1451d5257 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java
@@ -31,8 +31,6 @@ import org.apache.hudi.internal.schema.io.FileBasedInternalSchemaStorageManager;
import org.apache.hudi.internal.schema.utils.InternalSchemaUtils;
import org.apache.hudi.internal.schema.utils.SerDeHelper;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import com.github.benmanes.caffeine.cache.Cache;
@@ -174,19 +172,19 @@ public class InternalSchemaCache {
* step3:
* if we cannot parser internalSchema in step2 (eg: schema evolution is not enabled when we create hoodie table, however after some inserts we enable schema evolution)
* try to convert table schema to internalSchema.
- * @param versionId the internalSchema version to be search.
- * @param tablePath table path
- * @param storageConf conf
+ *
+ * @param versionId the internalSchema version to be search.
+ * @param tablePath table path
+ * @param storage {@link HoodieStorage} instance.
* @param validCommits current validate commits, use to make up the commit file path/verify the validity of the history schema files
* @return a internalSchema.
*/
- public static InternalSchema getInternalSchemaByVersionId(long versionId, String tablePath, StorageConfiguration<?> storageConf, String validCommits) {
+ public static InternalSchema getInternalSchemaByVersionId(long versionId, String tablePath, HoodieStorage storage, String validCommits) {
String avroSchema = "";
Set<String> commitSet = Arrays.stream(validCommits.split(",")).collect(Collectors.toSet());
List<String> validateCommitList =
commitSet.stream().map(HoodieInstant::extractTimestamp).collect(Collectors.toList());
- HoodieStorage storage = HoodieStorageUtils.getStorage(tablePath, storageConf);
StoragePath hoodieMetaPath = new StoragePath(tablePath, HoodieTableMetaClient.METAFOLDER_NAME);
//step1:
StoragePath candidateCommitFile = commitSet.stream()
@@ -215,7 +213,7 @@ public class InternalSchemaCache {
}
// step2:
FileBasedInternalSchemaStorageManager fileBasedInternalSchemaStorageManager =
- new FileBasedInternalSchemaStorageManager(storageConf, new StoragePath(tablePath));
+ new FileBasedInternalSchemaStorageManager(storage, new StoragePath(tablePath));
String latestHistorySchema =
fileBasedInternalSchemaStorageManager.getHistorySchemaStrByGivenValidCommits(validateCommitList);
if (latestHistorySchema.isEmpty()) {
@@ -234,7 +232,7 @@ public class InternalSchemaCache {
public static InternalSchema getInternalSchemaByVersionId(long versionId, HoodieTableMetaClient metaClient) {
String validCommitLists = metaClient
.getCommitsAndCompactionTimeline().filterCompletedInstants().getInstantsAsStream().map(HoodieInstant::getFileName).collect(Collectors.joining(","));
- return getInternalSchemaByVersionId(versionId, metaClient.getBasePathV2().toString(), metaClient.getStorageConf(), validCommitLists);
+ return getInternalSchemaByVersionId(versionId, metaClient.getBasePathV2().toString(), metaClient.getStorage(), validCommitLists);
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
index e5a58631059..88a2cbb1bff 100644
--- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
+++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
@@ -30,8 +30,6 @@ import org.apache.hudi.internal.schema.InternalSchema;
import org.apache.hudi.internal.schema.utils.InternalSchemaUtils;
import org.apache.hudi.internal.schema.utils.SerDeHelper;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.slf4j.Logger;
@@ -57,18 +55,18 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem
public static final String SCHEMA_NAME = ".schema";
private final StoragePath baseSchemaPath;
- private final StorageConfiguration<?> conf;
+ private final HoodieStorage storage;
private HoodieTableMetaClient metaClient;
- public FileBasedInternalSchemaStorageManager(StorageConfiguration<?> conf, StoragePath baseTablePath) {
+ public FileBasedInternalSchemaStorageManager(HoodieStorage storage, StoragePath baseTablePath) {
StoragePath metaPath = new StoragePath(baseTablePath, HoodieTableMetaClient.METAFOLDER_NAME);
this.baseSchemaPath = new StoragePath(metaPath, SCHEMA_NAME);
- this.conf = conf;
+ this.storage = storage;
}
public FileBasedInternalSchemaStorageManager(HoodieTableMetaClient metaClient) {
this.baseSchemaPath = new StoragePath(metaClient.getMetaPath(), SCHEMA_NAME);
- this.conf = metaClient.getStorageConf();
+ this.storage = metaClient.getStorage();
this.metaClient = metaClient;
}
@@ -77,7 +75,7 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem
if (metaClient == null) {
metaClient = HoodieTableMetaClient.builder()
.setBasePath(baseSchemaPath.getParent().getParent().toString())
- .setConf(conf.newInstance())
+ .setConf(storage.getConf().newInstance())
.setTimeGeneratorConfig(
HoodieTimeGeneratorConfig.defaultConfig(baseSchemaPath.getParent().getParent().toString()))
.build();
@@ -100,7 +98,6 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem
private void cleanResidualFiles() {
List<String> validateCommits = getValidInstants();
try {
- HoodieStorage storage = HoodieStorageUtils.getStorage(baseSchemaPath, conf);
if (storage.exists(baseSchemaPath)) {
List<String> candidateSchemaFiles = storage.listDirectEntries(baseSchemaPath).stream()
.filter(f -> f.isFile())
@@ -124,7 +121,6 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem
public void cleanOldFiles(List<String> validateCommits) {
try {
- HoodieStorage storage = HoodieStorageUtils.getStorage(baseSchemaPath, conf);
if (storage.exists(baseSchemaPath)) {
List<String> candidateSchemaFiles = storage.listDirectEntries(baseSchemaPath).stream()
.filter(f -> f.isFile())
@@ -155,7 +151,6 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem
public String getHistorySchemaStrByGivenValidCommits(List<String> validCommits) {
List<String> commitList = validCommits == null || validCommits.isEmpty() ? getValidInstants() : validCommits;
try {
- HoodieStorage storage = HoodieStorageUtils.getStorage(baseSchemaPath, conf);
if (storage.exists(baseSchemaPath)) {
List<String> validaSchemaFiles = storage.listDirectEntries(baseSchemaPath).stream()
.filter(f -> f.isFile() && f.getPath().getName().endsWith(SCHEMA_COMMIT_ACTION))
diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java
index 18dd976798d..0c63a3057c2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java
+++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java
@@ -24,7 +24,6 @@ import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -40,9 +39,9 @@ import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET;
*/
public class HoodieFileReaderFactory {
- protected final StorageConfiguration<?> storageConf;
- public HoodieFileReaderFactory(StorageConfiguration<?> storageConf) {
- this.storageConf = storageConf;
+ protected final HoodieStorage storage;
+ public HoodieFileReaderFactory(HoodieStorage storage) {
+ this.storage = storage;
}
public HoodieFileReader getFileReader(HoodieConfig hoodieConfig, StoragePath path) throws IOException {
diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java
index 569621fdf79..c96c6095582 100644
--- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java
+++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java
@@ -26,7 +26,7 @@ import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -39,24 +39,24 @@ import static org.apache.hudi.common.model.HoodieFileFormat.ORC;
import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET;
public class HoodieFileWriterFactory {
- protected final StorageConfiguration<?> storageConf;
+ protected final HoodieStorage storage;
- public HoodieFileWriterFactory(StorageConfiguration<?> storageConf) {
- this.storageConf = storageConf;
+ public HoodieFileWriterFactory(HoodieStorage storage) {
+ this.storage = storage;
}
public static <T, I, K, O> HoodieFileWriter getFileWriter(
- String instantTime, StoragePath path, StorageConfiguration<?> conf, HoodieConfig config, Schema schema,
+ String instantTime, StoragePath path, HoodieStorage storage, HoodieConfig config, Schema schema,
TaskContextSupplier taskContextSupplier, HoodieRecordType recordType) throws IOException {
final String extension = FSUtils.getFileExtension(path.getName());
- HoodieFileWriterFactory factory = HoodieIOFactory.getIOFactory(conf).getWriterFactory(recordType);
+ HoodieFileWriterFactory factory = HoodieIOFactory.getIOFactory(storage).getWriterFactory(recordType);
return factory.getFileWriterByFormat(extension, instantTime, path, config, schema, taskContextSupplier);
}
public static <T, I, K, O> HoodieFileWriter getFileWriter(HoodieFileFormat format, OutputStream outputStream,
- StorageConfiguration<?> conf, HoodieConfig config, Schema schema, HoodieRecordType recordType)
+ HoodieStorage storage, HoodieConfig config, Schema schema, HoodieRecordType recordType)
throws IOException {
- HoodieFileWriterFactory factory = HoodieIOFactory.getIOFactory(conf).getWriterFactory(recordType);
+ HoodieFileWriterFactory factory = HoodieIOFactory.getIOFactory(storage).getWriterFactory(recordType);
return factory.getFileWriterByFormat(format, outputStream, config, schema);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieIOFactory.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieIOFactory.java
index 3ae6b60321e..cba3c7b0e98 100644
--- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieIOFactory.java
+++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieIOFactory.java
@@ -25,25 +25,24 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
/**
* Base class to get HoodieFileReaderFactory and HoodieFileWriterFactory
*/
public abstract class HoodieIOFactory {
- protected final StorageConfiguration<?> storageConf;
+ protected final HoodieStorage storage;
- public HoodieIOFactory(StorageConfiguration<?> storageConf) {
- this.storageConf = storageConf;
+ public HoodieIOFactory(HoodieStorage storage) {
+ this.storage = storage;
}
- public static HoodieIOFactory getIOFactory(StorageConfiguration<?> storageConf) {
- String ioFactoryClass = storageConf.getString(HoodieStorageConfig.HOODIE_IO_FACTORY_CLASS.key())
+ public static HoodieIOFactory getIOFactory(HoodieStorage storage) {
+ String ioFactoryClass = storage.getConf().getString(HoodieStorageConfig.HOODIE_IO_FACTORY_CLASS.key())
.orElse(HoodieStorageConfig.HOODIE_IO_FACTORY_CLASS.defaultValue());
try {
return (HoodieIOFactory) ReflectionUtils
- .loadClass(ioFactoryClass, new Class<?>[] {StorageConfiguration.class}, storageConf);
+ .loadClass(ioFactoryClass, new Class<?>[] {HoodieStorage.class}, storage);
} catch (Exception e) {
throw new HoodieException("Unable to create " + ioFactoryClass, e);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java
index fef595d7923..23fc8dba327 100644
--- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java
+++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java
@@ -38,8 +38,6 @@ import org.apache.hudi.io.hfile.HFileReaderImpl;
import org.apache.hudi.io.hfile.KeyValue;
import org.apache.hudi.io.hfile.UTF8StringKey;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.util.Lazy;
@@ -69,14 +67,14 @@ import static org.apache.hudi.io.hfile.HFileUtils.isPrefixOfKey;
public class HoodieNativeAvroHFileReader extends HoodieAvroHFileReaderImplBase {
private static final Logger LOG = LoggerFactory.getLogger(HoodieNativeAvroHFileReader.class);
- private final StorageConfiguration<?> conf;
+ private final HoodieStorage storage;
private final Option<StoragePath> path;
private final Option<byte[]> bytesContent;
private Option<HFileReader> sharedHFileReader;
private final Lazy<Schema> schema;
- public HoodieNativeAvroHFileReader(StorageConfiguration<?> conf, StoragePath path, Option<Schema> schemaOption) {
- this.conf = conf;
+ public HoodieNativeAvroHFileReader(HoodieStorage storage, StoragePath path, Option<Schema> schemaOption) {
+ this.storage = storage;
this.path = Option.of(path);
this.bytesContent = Option.empty();
this.sharedHFileReader = Option.empty();
@@ -84,8 +82,8 @@ public class HoodieNativeAvroHFileReader extends HoodieAvroHFileReaderImplBase {
.orElseGet(() -> Lazy.lazily(() -> fetchSchema(getSharedHFileReader())));
}
- public HoodieNativeAvroHFileReader(StorageConfiguration<?> conf, byte[] content, Option<Schema> schemaOption) {
- this.conf = conf;
+ public HoodieNativeAvroHFileReader(HoodieStorage storage, byte[] content, Option<Schema> schemaOption) {
+ this.storage = storage;
this.path = Option.empty();
this.bytesContent = Option.of(content);
this.sharedHFileReader = Option.empty();
@@ -261,7 +259,6 @@ public class HoodieNativeAvroHFileReader extends HoodieAvroHFileReaderImplBase {
SeekableDataInputStream inputStream;
long fileSize;
if (path.isPresent()) {
- HoodieStorage storage = HoodieStorageUtils.getStorage(path.get(), conf);
fileSize = storage.getPathInfo(path.get()).getLength();
inputStream = storage.openSeekable(path.get(), false);
} else {
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/AbstractHoodieTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/AbstractHoodieTableMetadata.java
index 9128b82a3c5..ea61cd1130e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/AbstractHoodieTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/AbstractHoodieTableMetadata.java
@@ -24,6 +24,7 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.expression.ArrayData;
import org.apache.hudi.internal.schema.Type;
import org.apache.hudi.internal.schema.Types;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
@@ -35,6 +36,7 @@ import java.util.stream.IntStream;
public abstract class AbstractHoodieTableMetadata implements HoodieTableMetadata {
protected transient HoodieEngineContext engineContext;
+ protected transient HoodieStorage storage;
protected final StorageConfiguration<?> storageConf;
protected final StoragePath dataBasePath;
@@ -42,9 +44,10 @@ public abstract class AbstractHoodieTableMetadata implements HoodieTableMetadata
// TODO get this from HoodieConfig
protected final boolean caseSensitive = false;
- public AbstractHoodieTableMetadata(HoodieEngineContext engineContext, StorageConfiguration<?> conf, String dataBasePath) {
+ public AbstractHoodieTableMetadata(HoodieEngineContext engineContext, HoodieStorage storage, String dataBasePath) {
this.engineContext = engineContext;
- this.storageConf = conf;
+ this.storage = storage;
+ this.storageConf = storage.getConf();
this.dataBasePath = new StoragePath(dataBasePath);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
index 31c6188184a..7c340d27714 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
@@ -39,10 +39,9 @@ import org.apache.hudi.common.util.hash.ColumnIndexID;
import org.apache.hudi.common.util.hash.FileIndexID;
import org.apache.hudi.common.util.hash.PartitionIndexID;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -79,11 +78,14 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
protected final boolean hiveStylePartitioningEnabled;
protected final boolean urlEncodePartitioningEnabled;
- protected BaseTableMetadata(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig, String dataBasePath) {
- super(engineContext, engineContext.getStorageConf(), dataBasePath);
+ protected BaseTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig,
+ String dataBasePath) {
+ super(engineContext, storage, dataBasePath);
this.dataMetaClient = HoodieTableMetaClient.builder()
- .setConf(storageConf.newInstance())
+ .setConf(storage.getConf().newInstance())
.setBasePath(dataBasePath)
.build();
@@ -94,7 +96,7 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
if (metadataConfig.isMetricsEnabled()) {
this.metrics = Option.of(new HoodieMetadataMetrics(HoodieMetricsConfig.newBuilder()
- .fromProperties(metadataConfig.getProps()).build(), getStorageConf()));
+ .fromProperties(metadataConfig.getProps()).build(), dataMetaClient.getStorage()));
} else {
this.metrics = Option.empty();
}
@@ -356,9 +358,9 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
HoodieMetadataPayload metadataPayload = record.getData();
checkForSpuriousDeletes(metadataPayload, recordKey);
try {
- return metadataPayload.getFileList(getStorageConf(), partitionPath);
- } catch (IOException e) {
- throw new HoodieIOException("Failed to extract file-pathInfoList from the payload", e);
+ return metadataPayload.getFileList(dataMetaClient.getStorage(), partitionPath);
+ } catch (Exception e) {
+ throw new HoodieException("Failed to extract file-pathInfoList from the payload", e);
}
})
.orElseGet(Collections::emptyList);
@@ -386,9 +388,6 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
metrics.ifPresent(
m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_FILES_STR, timer.endTimer()));
- HoodieStorage storage =
- HoodieStorageUtils.getStorage(partitionPaths.get(0), getStorageConf());
-
Map<String, List<StoragePathInfo>> partitionPathToFilesMap =
partitionIdRecordPairs.entrySet().stream()
.map(e -> {
@@ -398,7 +397,7 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
HoodieMetadataPayload metadataPayload = e.getValue().getData();
checkForSpuriousDeletes(metadataPayload, partitionId);
- List<StoragePathInfo> files = metadataPayload.getFileList(storage, partitionPath);
+ List<StoragePathInfo> files = metadataPayload.getFileList(dataMetaClient.getStorage(), partitionPath);
return Pair.of(partitionPath.toString(), files);
})
.collect(Collectors.toMap(Pair::getKey, Pair::getValue));
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
index 99484cce5ff..1fd9d0f5696 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
@@ -41,7 +41,6 @@ import org.apache.hudi.expression.Predicates;
import org.apache.hudi.internal.schema.Types;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -67,19 +66,18 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
private final boolean urlEncodePartitioningEnabled;
public FileSystemBackedTableMetadata(HoodieEngineContext engineContext, HoodieTableConfig tableConfig,
- StorageConfiguration<?> conf, String datasetBasePath) {
- super(engineContext, conf, datasetBasePath);
+ HoodieStorage storage, String datasetBasePath) {
+ super(engineContext, storage, datasetBasePath);
this.hiveStylePartitioningEnabled = Boolean.parseBoolean(tableConfig.getHiveStylePartitioningEnable());
this.urlEncodePartitioningEnabled = Boolean.parseBoolean(tableConfig.getUrlEncodePartitioning());
}
public FileSystemBackedTableMetadata(HoodieEngineContext engineContext,
- StorageConfiguration<?> conf,
+ HoodieStorage storage,
String datasetBasePath) {
- super(engineContext, conf, datasetBasePath);
+ super(engineContext, storage, datasetBasePath);
- HoodieStorage storage = HoodieStorageUtils.getStorage(dataBasePath, conf);
StoragePath metaPath =
new StoragePath(dataBasePath, HoodieTableMetaClient.METAFOLDER_NAME);
TableNotFoundException.checkTableValidity(storage, this.dataBasePath, metaPath);
@@ -90,10 +88,16 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
Boolean.parseBoolean(tableConfig.getUrlEncodePartitioning());
}
+ public HoodieStorage getStorage() {
+ if (storage == null) {
+ storage = HoodieStorageUtils.getStorage(dataBasePath, storageConf);
+ }
+ return storage;
+ }
+
@Override
public List<StoragePathInfo> getAllFilesInPartition(StoragePath partitionPath) throws IOException {
- HoodieStorage storage = HoodieStorageUtils.getStorage(partitionPath, storageConf);
- return FSUtils.getAllDataFilesInPartition(storage, partitionPath);
+ return FSUtils.getAllDataFilesInPartition(getStorage(), partitionPath);
}
@Override
@@ -166,8 +170,7 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
// Need to use serializable file status here, see HUDI-5936
List<StoragePathInfo> dirToFileListing = engineContext.flatMap(pathsToList, path -> {
try {
- HoodieStorage storage = HoodieStorageUtils.getStorage(path, storageConf);
- return storage.listDirectEntries(path).stream();
+ return getStorage().listDirectEntries(path).stream();
} catch (FileNotFoundException e) {
// The partition may have been cleaned.
return Stream.empty();
@@ -186,9 +189,8 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
engineContext.map(dirToFileListing,
fileInfo -> {
StoragePath path = fileInfo.getPath();
- HoodieStorage storage = HoodieStorageUtils.getStorage(path, storageConf);
if (fileInfo.isDirectory()) {
- if (HoodiePartitionMetadata.hasPartitionMetadata(storage, path)) {
+ if (HoodiePartitionMetadata.hasPartitionMetadata(getStorage(), path)) {
return Pair.of(
Option.of(FSUtils.getRelativePartitionPath(dataBasePath,
path)),
@@ -257,9 +259,8 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
engineContext.map(new ArrayList<>(partitionPaths),
partitionPathStr -> {
StoragePath partitionPath = new StoragePath(partitionPathStr);
- HoodieStorage storage = HoodieStorageUtils.getStorage(partitionPath, storageConf);
return Pair.of(partitionPathStr,
- FSUtils.getAllDataFilesInPartition(storage, partitionPath));
+ FSUtils.getAllDataFilesInPartition(getStorage(), partitionPath));
}, parallelism);
return partitionToFiles.stream().collect(Collectors.toMap(pair -> pair.getLeft(),
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
index 78a3ea48e38..acaaa86c9c2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
@@ -52,6 +52,7 @@ import org.apache.hudi.expression.Expression;
import org.apache.hudi.internal.schema.Types;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.io.storage.HoodieSeekingFileReader;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.util.Transient;
@@ -103,13 +104,18 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
// Latest file slices in the metadata partitions
private final Map<String, List<FileSlice>> partitionFileSliceMap = new ConcurrentHashMap<>();
- public HoodieBackedTableMetadata(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig, String datasetBasePath) {
- this(engineContext, metadataConfig, datasetBasePath, false);
+ public HoodieBackedTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig,
+ String datasetBasePath) {
+ this(engineContext, storage, metadataConfig, datasetBasePath, false);
}
- public HoodieBackedTableMetadata(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig,
+ public HoodieBackedTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig,
String datasetBasePath, boolean reuse) {
- super(engineContext, metadataConfig, datasetBasePath);
+ super(engineContext, storage, metadataConfig, datasetBasePath);
this.reuse = reuse;
this.metadataBasePath = HoodieTableMetadata.getMetadataTableBasePath(dataBasePath.toString());
@@ -606,7 +612,8 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
Option<HoodieBaseFile> basefile = slice.getBaseFile();
if (basefile.isPresent()) {
StoragePath baseFilePath = basefile.get().getStoragePath();
- baseFileReader = (HoodieSeekingFileReader<?>) HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecordType.AVRO)
+ baseFileReader = (HoodieSeekingFileReader<?>) HoodieIOFactory.getIOFactory(metadataMetaClient.getStorage())
+ .getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, baseFilePath);
baseFileOpenMs = timer.endTimer();
LOG.info(String.format("Opened metadata base file from %s at instant %s in %d ms", baseFilePath,
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
index 05b3efa9104..624d6099833 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
@@ -25,8 +25,8 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import java.io.IOException;
import java.util.List;
@@ -52,7 +52,8 @@ public class HoodieMetadataFileSystemView extends HoodieTableFileSystemView {
HoodieTableMetaClient metaClient,
HoodieTimeline visibleActiveTimeline,
HoodieMetadataConfig metadataConfig) {
- this(metaClient, visibleActiveTimeline, HoodieTableMetadata.create(engineContext, metadataConfig, metaClient.getBasePath(), true));
+ this(metaClient, visibleActiveTimeline, HoodieTableMetadata.create(
+ engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePath(), true));
}
/**
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
index fce32753883..25a79cb12d8 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
@@ -27,7 +27,7 @@ import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.metrics.HoodieGauge;
import org.apache.hudi.metrics.Metrics;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import com.codahale.metrics.MetricRegistry;
import org.slf4j.Logger;
@@ -81,8 +81,8 @@ public class HoodieMetadataMetrics implements Serializable {
private final transient MetricRegistry metricsRegistry;
private final transient Metrics metrics;
- public HoodieMetadataMetrics(HoodieMetricsConfig metricsConfig, StorageConfiguration<?> storageConf) {
- this.metrics = Metrics.getInstance(metricsConfig, storageConf);
+ public HoodieMetadataMetrics(HoodieMetricsConfig metricsConfig, HoodieStorage storage) {
+ this.metrics = Metrics.getInstance(metricsConfig, storage);
this.metricsRegistry = metrics.getRegistry();
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
index a0be17fb49a..beecf35aa55 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
@@ -39,8 +39,6 @@ import org.apache.hudi.common.util.hash.PartitionIndexID;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.util.Lazy;
@@ -493,15 +491,6 @@ public class HoodieMetadataPayload implements HoodieRecordPayload<HoodieMetadata
return Option.of(columnStatMetadata);
}
- /**
- * Returns the files added as part of this record.
- */
- public List<StoragePathInfo> getFileList(StorageConfiguration<?> storageConf, StoragePath partitionPath)
- throws IOException {
- HoodieStorage storage = HoodieStorageUtils.getStorage(partitionPath, storageConf);
- return getFileList(storage, partitionPath);
- }
-
/**
* Returns the files added as part of this record.
*/
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
index f389f151b5a..40479c548d1 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
@@ -31,6 +31,7 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.expression.Expression;
import org.apache.hudi.internal.schema.Types;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -111,33 +112,42 @@ public interface HoodieTableMetadata extends Serializable, AutoCloseable {
return basePath.endsWith(HoodieTableMetaClient.METADATA_TABLE_FOLDER_PATH);
}
- static HoodieTableMetadata create(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig, String datasetBasePath) {
- return create(engineContext, metadataConfig, datasetBasePath, false);
+ static HoodieTableMetadata create(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig,
+ String datasetBasePath) {
+ return create(engineContext, storage, metadataConfig, datasetBasePath, false);
}
- static HoodieTableMetadata create(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig, String datasetBasePath, boolean reuse) {
+ static HoodieTableMetadata create(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig,
+ String datasetBasePath,
+ boolean reuse) {
if (metadataConfig.isEnabled()) {
- HoodieBackedTableMetadata metadata = createHoodieBackedTableMetadata(engineContext, metadataConfig, datasetBasePath, reuse);
+ HoodieBackedTableMetadata metadata = createHoodieBackedTableMetadata(engineContext, storage, metadataConfig, datasetBasePath, reuse);
// If the MDT is not initialized then we fallback to FSBackedTableMetadata
if (metadata.isMetadataTableInitialized()) {
return metadata;
}
}
- return createFSBackedTableMetadata(engineContext, datasetBasePath);
+ return createFSBackedTableMetadata(engineContext, storage, datasetBasePath);
}
static FileSystemBackedTableMetadata createFSBackedTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
String datasetBasePath) {
return new FileSystemBackedTableMetadata(
- engineContext, engineContext.getStorageConf().newInstance(), datasetBasePath);
+ engineContext, storage, datasetBasePath);
}
static HoodieBackedTableMetadata createHoodieBackedTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
HoodieMetadataConfig metadataConfig,
String datasetBasePath,
boolean reuse) {
- return new HoodieBackedTableMetadata(engineContext, metadataConfig, datasetBasePath, reuse);
+ return new HoodieBackedTableMetadata(engineContext, storage, metadataConfig, datasetBasePath, reuse);
}
/**
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
index a274d07b027..60bfaa3c878 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
@@ -517,7 +517,7 @@ public class HoodieTableMetadataUtil {
}
final StoragePath writeFilePath = new StoragePath(dataMetaClient.getBasePathV2(), pathWithPartition);
- try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(dataMetaClient.getStorageConf())
+ try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(dataMetaClient.getStorage())
.getReaderFactory(HoodieRecordType.AVRO).getFileReader(hoodieConfig, writeFilePath)) {
try {
final BloomFilter fileBloomFilter = fileReader.readBloomFilter();
@@ -904,7 +904,7 @@ public class HoodieTableMetadataUtil {
if (!isDeleted) {
final String pathWithPartition = partitionName + "/" + filename;
final StoragePath addedFilePath = new StoragePath(dataMetaClient.getBasePathV2(), pathWithPartition);
- bloomFilterBuffer = readBloomFilter(dataMetaClient.getStorageConf(), addedFilePath);
+ bloomFilterBuffer = readBloomFilter(dataMetaClient.getStorage(), addedFilePath);
// If reading the bloom filter failed then do not add a record for this file
if (bloomFilterBuffer == null) {
@@ -959,9 +959,9 @@ public class HoodieTableMetadataUtil {
});
}
- private static ByteBuffer readBloomFilter(StorageConfiguration<?> conf, StoragePath filePath) throws IOException {
- HoodieConfig hoodieConfig = getReaderConfigs(conf);
- try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(conf).getReaderFactory(HoodieRecordType.AVRO)
+ private static ByteBuffer readBloomFilter(HoodieStorage storage, StoragePath filePath) throws IOException {
+ HoodieConfig hoodieConfig = getReaderConfigs(storage.getConf());
+ try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, filePath)) {
final BloomFilter fileBloomFilter = fileReader.readBloomFilter();
if (fileBloomFilter == null) {
@@ -1214,7 +1214,7 @@ public class HoodieTableMetadataUtil {
if (filePath.endsWith(HoodieFileFormat.PARQUET.getFileExtension())) {
StoragePath fullFilePath = new StoragePath(datasetMetaClient.getBasePathV2(), filePath);
return FileFormatUtils.getInstance(HoodieFileFormat.PARQUET)
- .readColumnStatsFromMetadata(datasetMetaClient.getStorageConf(), fullFilePath, columnsToIndex);
+ .readColumnStatsFromMetadata(datasetMetaClient.getStorage(), fullFilePath, columnsToIndex);
}
LOG.warn("Column range index not supported for: {}", filePath);
@@ -1476,8 +1476,7 @@ public class HoodieTableMetadataUtil {
public static String deleteMetadataTable(HoodieTableMetaClient dataMetaClient, HoodieEngineContext context, boolean backup) {
final StoragePath metadataTablePath =
HoodieTableMetadata.getMetadataTableBasePath(dataMetaClient.getBasePathV2());
- HoodieStorage storage = HoodieStorageUtils.getStorage(
- metadataTablePath.toString(), context.getStorageConf());
+ HoodieStorage storage = dataMetaClient.getStorage();
dataMetaClient.getTableConfig().clearMetadataPartitions(dataMetaClient);
try {
if (!storage.exists(metadataTablePath)) {
@@ -1532,7 +1531,7 @@ public class HoodieTableMetadataUtil {
}
final StoragePath metadataTablePartitionPath = new StoragePath(HoodieTableMetadata.getMetadataTableBasePath(dataMetaClient.getBasePathV2()), partitionPath);
- HoodieStorage storage = HoodieStorageUtils.getStorage(metadataTablePartitionPath, context.getStorageConf());
+ HoodieStorage storage = dataMetaClient.getStorage();
dataMetaClient.getTableConfig().setMetadataPartitionState(dataMetaClient, partitionPath, false);
try {
if (!storage.exists(metadataTablePartitionPath)) {
@@ -1764,7 +1763,8 @@ public class HoodieTableMetadataUtil {
final String fileId = baseFile.getFileId();
final String instantTime = baseFile.getCommitTime();
- HoodieFileReader reader = HoodieIOFactory.getIOFactory(configuration).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
+ HoodieFileReader reader = HoodieIOFactory.getIOFactory(HoodieStorageUtils.getStorage(basePath, configuration))
+ .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(config, dataFilePath);
return getHoodieRecordIterator(reader.getRecordKeyIterator(), forDelete, partition, fileId, instantTime);
});
@@ -1825,7 +1825,8 @@ public class HoodieTableMetadataUtil {
final String fileId = baseFile.getFileId();
final String instantTime = baseFile.getCommitTime();
HoodieConfig hoodieConfig = getReaderConfigs(storageConf);
- HoodieFileReader reader = HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
+ HoodieFileReader reader = HoodieIOFactory.getIOFactory(metaClient.getStorage())
+ .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, dataFilePath);
return getHoodieRecordIterator(reader.getRecordKeyIterator(), forDelete, partition, fileId, instantTime);
});
diff --git a/hudi-common/src/main/java/org/apache/hudi/metrics/Metrics.java b/hudi-common/src/main/java/org/apache/hudi/metrics/Metrics.java
index cc50d3a4147..441c0b5b002 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metrics/Metrics.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metrics/Metrics.java
@@ -24,8 +24,6 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import com.codahale.metrics.MetricRegistry;
@@ -53,10 +51,10 @@ public class Metrics {
private final String basePath;
private boolean initialized = false;
private transient Thread shutdownThread = null;
- private final StorageConfiguration<?> storageConf;
+ private final HoodieStorage storage;
- public Metrics(HoodieMetricsConfig metricConfig, StorageConfiguration<?> storageConf) {
- this.storageConf = storageConf;
+ public Metrics(HoodieMetricsConfig metricConfig, HoodieStorage storage) {
+ this.storage = storage;
registry = new MetricRegistry();
commonMetricPrefix = metricConfig.getMetricReporterMetricsNamePrefix();
reporters = new ArrayList<>();
@@ -80,13 +78,13 @@ public class Metrics {
registerGauges(Registry.getAllMetrics(true, true), Option.of(commonMetricPrefix));
}
- public static synchronized Metrics getInstance(HoodieMetricsConfig metricConfig, StorageConfiguration<?> storageConf) {
+ public static synchronized Metrics getInstance(HoodieMetricsConfig metricConfig, HoodieStorage storage) {
String basePath = getBasePath(metricConfig);
if (METRICS_INSTANCE_PER_BASEPATH.containsKey(basePath)) {
return METRICS_INSTANCE_PER_BASEPATH.get(basePath);
}
- Metrics metrics = new Metrics(metricConfig, storageConf);
+ Metrics metrics = new Metrics(metricConfig, storage);
METRICS_INSTANCE_PER_BASEPATH.put(basePath, metrics);
return metrics;
}
@@ -100,10 +98,10 @@ public class Metrics {
private List<MetricsReporter> addAdditionalMetricsExporters(HoodieMetricsConfig metricConfig) {
List<MetricsReporter> reporterList = new ArrayList<>();
List<String> propPathList = StringUtils.split(metricConfig.getMetricReporterFileBasedConfigs(), ",");
- try (HoodieStorage storage = HoodieStorageUtils.getStorage(propPathList.get(0), storageConf)) {
+ try (HoodieStorage newStorage = storage.newInstance(new StoragePath(propPathList.get(0)), storage.getConf())) {
for (String propPath : propPathList) {
HoodieMetricsConfig secondarySourceConfig = HoodieMetricsConfig.newBuilder().fromInputStream(
- storage.open(new StoragePath(propPath))).withPath(metricConfig.getBasePath()).build();
+ newStorage.open(new StoragePath(propPath))).withPath(metricConfig.getBasePath()).build();
Option<MetricsReporter> reporter = MetricsReporterFactory.createReporter(secondarySourceConfig, registry);
if (reporter.isPresent()) {
reporterList.add(reporter.get());
diff --git a/hudi-common/src/main/java/org/apache/hudi/storage/HoodieStorageUtils.java b/hudi-common/src/main/java/org/apache/hudi/storage/HoodieStorageUtils.java
index 770fc77372e..6946034adf4 100644
--- a/hudi-common/src/main/java/org/apache/hudi/storage/HoodieStorageUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/storage/HoodieStorageUtils.java
@@ -19,7 +19,9 @@
package org.apache.hudi.storage;
-import static org.apache.hudi.io.storage.HoodieIOFactory.getIOFactory;
+import org.apache.hudi.common.config.HoodieStorageConfig;
+import org.apache.hudi.common.util.ReflectionUtils;
+import org.apache.hudi.exception.HoodieException;
public class HoodieStorageUtils {
public static final String DEFAULT_URI = "file:///";
@@ -33,6 +35,13 @@ public class HoodieStorageUtils {
}
public static HoodieStorage getStorage(StoragePath path, StorageConfiguration<?> conf) {
- return getIOFactory(conf).getStorage(path);
+ String storageClass = conf.getString(HoodieStorageConfig.HOODIE_STORAGE_CLASS.key())
+ .orElse(HoodieStorageConfig.HOODIE_STORAGE_CLASS.defaultValue());
+ try {
+ return (HoodieStorage) ReflectionUtils.loadClass(
+ storageClass, new Class<?>[] {StoragePath.class, StorageConfiguration.class}, path, conf);
+ } catch (Exception e) {
+ throw new HoodieException("Unable to create " + storageClass, e);
+ }
}
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java
index b492302e7d9..ec3457801f7 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java
@@ -153,7 +153,7 @@ public abstract class TestHoodieFileGroupReaderBase<T> {
engineContext, FileSystemViewStorageConfig.newBuilder().build(),
HoodieCommonConfig.newBuilder().build(),
mc -> HoodieTableMetadata.create(
- engineContext, metadataConfig, mc.getBasePathV2().toString()));
+ engineContext, mc.getStorage(), metadataConfig, mc.getBasePathV2().toString()));
SyncableFileSystemView fsView = viewManager.getFileSystemView(metaClient);
FileSlice fileSlice = fsView.getAllFileSlices(partitionPaths[0]).findFirst().get();
List<String> logFilePathList = getLogFileListFromFileSlice(fileSlice);
@@ -170,7 +170,7 @@ public abstract class TestHoodieFileGroupReaderBase<T> {
assertEquals(containsBaseFile, fileSlice.getBaseFile().isPresent());
HoodieFileGroupReader<T> fileGroupReader = new HoodieFileGroupReader<>(
getHoodieReaderContext(tablePath, avroSchema),
- storageConf,
+ metaClient.getStorage(),
tablePath,
metaClient.getActiveTimeline().lastInstant().get().getTimestamp(),
fileSlice,
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
index e25a379f2c2..4d4fc57cc09 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
@@ -56,6 +56,8 @@ import java.util.Properties;
import java.util.UUID;
import java.util.stream.Collectors;
+import static org.apache.hudi.storage.HoodieStorageUtils.DEFAULT_URI;
+
/**
* A utility class for testing.
*/
@@ -78,6 +80,10 @@ public class HoodieTestUtils {
new Class<?>[] {Boolean.class}, true);
}
+ public static HoodieStorage getDefaultStorage() {
+ return getStorage(DEFAULT_URI);
+ }
+
public static HoodieStorage getStorage(String path) {
return HoodieStorageUtils.getStorage(path, getDefaultStorageConf());
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestUtils.java
index 6bd4b7eabf4..49acf352628 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestUtils.java
@@ -27,7 +27,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.read.HoodieFileGroupReader;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ExternalSpillableMap;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
@@ -42,14 +42,14 @@ public class HoodieFileGroupReaderTestUtils {
long start,
long length,
TypedProperties properties,
- StorageConfiguration<?> storageConf,
+ HoodieStorage storage,
HoodieTableConfig tableConfig,
HoodieReaderContext<IndexedRecord> readerContext
) {
assert (fileSliceOpt.isPresent());
return new HoodieFileGroupReaderBuilder()
.withReaderContext(readerContext)
- .withStorageConf(storageConf)
+ .withStorage(storage)
.withFileSlice(fileSliceOpt.get())
.withStart(start)
.withLength(length)
@@ -61,7 +61,7 @@ public class HoodieFileGroupReaderTestUtils {
public static class HoodieFileGroupReaderBuilder {
private HoodieReaderContext<IndexedRecord> readerContext;
private FileSlice fileSlice;
- private StorageConfiguration<?> storageConf;
+ private HoodieStorage storage;
private TypedProperties props;
private long start;
private long length;
@@ -78,8 +78,8 @@ public class HoodieFileGroupReaderTestUtils {
return this;
}
- public HoodieFileGroupReaderBuilder withStorageConf(StorageConfiguration<?> storageConf) {
- this.storageConf = storageConf;
+ public HoodieFileGroupReaderBuilder withStorage(HoodieStorage storage) {
+ this.storage = storage;
return this;
}
@@ -113,7 +113,7 @@ public class HoodieFileGroupReaderTestUtils {
) {
return new HoodieFileGroupReader<>(
readerContext,
- storageConf,
+ storage,
basePath,
latestCommitTime,
fileSlice,
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java
index 2a736709b42..2ca33678adc 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java
@@ -49,12 +49,10 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.io.storage.HoodieAvroFileWriter;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
@@ -246,7 +244,7 @@ public class HoodieFileSliceTestUtils {
Schema schema,
String baseInstantTime
) throws IOException {
- StorageConfiguration<Configuration> conf = HoodieTestUtils.getDefaultStorageConfWithDefaults();
+ HoodieStorage storage = HoodieTestUtils.getStorage(baseFilePath);
// TODO: Optimize these hard-coded parameters for test purpose. (HUDI-7214)
HoodieConfig cfg = new HoodieConfig();
@@ -269,7 +267,7 @@ public class HoodieFileSliceTestUtils {
cfg.setValue(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED.key(), "true");
try (HoodieAvroFileWriter writer = (HoodieAvroFileWriter) HoodieFileWriterFactory
- .getFileWriter(baseInstantTime, new StoragePath(baseFilePath), conf, cfg,
+ .getFileWriter(baseInstantTime, new StoragePath(baseFilePath), storage, cfg,
schema, new LocalTaskContextSupplier(), HoodieRecord.HoodieRecordType.AVRO)) {
for (IndexedRecord record : records) {
writer.writeAvro(
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieTestReaderContext.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieTestReaderContext.java
index 0a925308836..08373882773 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieTestReaderContext.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieTestReaderContext.java
@@ -79,9 +79,9 @@ public class HoodieTestReaderContext extends HoodieReaderContext<IndexedRecord>
long length,
Schema dataSchema,
Schema requiredSchema,
- StorageConfiguration<?> conf
+ HoodieStorage storage
) throws IOException {
- HoodieAvroFileReader reader = (HoodieAvroFileReader) HoodieIOFactory.getIOFactory(conf)
+ HoodieAvroFileReader reader = (HoodieAvroFileReader) HoodieIOFactory.getIOFactory(storage)
.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO).getFileReader(new HoodieConfig(),
filePath, HoodieFileFormat.PARQUET, Option.empty());
return reader.getIndexedRecordIterator(dataSchema, requiredSchema);
diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/HoodieBackedTestDelayedTableMetadata.java b/hudi-common/src/test/java/org/apache/hudi/metadata/HoodieBackedTestDelayedTableMetadata.java
index b195083a201..51a300ef582 100644
--- a/hudi-common/src/test/java/org/apache/hudi/metadata/HoodieBackedTestDelayedTableMetadata.java
+++ b/hudi-common/src/test/java/org/apache/hudi/metadata/HoodieBackedTestDelayedTableMetadata.java
@@ -21,6 +21,7 @@ package org.apache.hudi.metadata;
import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.storage.HoodieStorage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,10 +34,11 @@ public class HoodieBackedTestDelayedTableMetadata extends HoodieBackedTableMetad
private static final Logger LOG = LoggerFactory.getLogger(HoodieBackedTestDelayedTableMetadata.class);
public HoodieBackedTestDelayedTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
HoodieMetadataConfig metadataConfig,
String datasetBasePath,
boolean reuse) {
- super(engineContext, metadataConfig, datasetBasePath, reuse);
+ super(engineContext, storage, metadataConfig, datasetBasePath, reuse);
}
@Override
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/BootstrapOperator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/BootstrapOperator.java
index ecd331079a1..0ab008ee0ac 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/BootstrapOperator.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/BootstrapOperator.java
@@ -38,7 +38,6 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.configuration.HadoopConfigurations;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.sink.bootstrap.aggregate.BootstrapAggFunction;
import org.apache.hudi.sink.meta.CkpMetadata;
import org.apache.hudi.sink.meta.CkpMetadataFactory;
@@ -145,7 +144,8 @@ public class BootstrapOperator<I, O extends HoodieRecord<?>>
String basePath = hoodieTable.getMetaClient().getBasePath();
int taskID = getRuntimeContext().getIndexOfThisSubtask();
LOG.info("Start loading records in table {} into the index state, taskId = {}", basePath, taskID);
- for (String partitionPath : FSUtils.getAllPartitionPaths(new HoodieFlinkEngineContext(hadoopConf), metadataConfig(conf), basePath)) {
+ for (String partitionPath : FSUtils.getAllPartitionPaths(
+ new HoodieFlinkEngineContext(hadoopConf), hoodieTable.getStorage(), metadataConfig(conf), basePath)) {
if (pattern.matcher(partitionPath).matches()) {
loadRecords(partitionPath);
}
@@ -222,7 +222,7 @@ public class BootstrapOperator<I, O extends HoodieRecord<?>>
return;
}
try (ClosableIterator<HoodieKey> iterator = fileUtils.getHoodieKeyIterator(
- HadoopFSUtils.getStorageConf(this.hadoopConf), baseFile.getStoragePath())) {
+ hoodieTable.getStorage(), baseFile.getStoragePath())) {
iterator.forEachRemaining(hoodieKey -> {
output.collect(new StreamRecord(new IndexRecord(generateHoodieRecord(hoodieKey, fileSlice))));
});
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java
index 460e36154cf..2e1bfd604d7 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java
@@ -274,11 +274,11 @@ public class ClusteringOperator extends TableStreamOperator<ClusteringCommitEven
try {
Option<HoodieFileReader> baseFileReader = StringUtils.isNullOrEmpty(clusteringOp.getDataFilePath())
? Option.empty()
- : Option.of(HoodieIOFactory.getIOFactory(table.getStorageConf())
+ : Option.of(HoodieIOFactory.getIOFactory(table.getStorage())
.getReaderFactory(table.getConfig().getRecordMerger().getRecordType())
.getFileReader(table.getConfig(), new StoragePath(clusteringOp.getDataFilePath())));
HoodieMergedLogRecordScanner scanner = HoodieMergedLogRecordScanner.newBuilder()
- .withStorage(table.getMetaClient().getStorage())
+ .withStorage(table.getStorage())
.withBasePath(table.getMetaClient().getBasePath())
.withLogFilePaths(clusteringOp.getDeltaFilePaths())
.withReaderSchema(readerSchema)
@@ -322,7 +322,7 @@ public class ClusteringOperator extends TableStreamOperator<ClusteringCommitEven
List<Iterator<RowData>> iteratorsForPartition = clusteringOps.stream().map(clusteringOp -> {
Iterable<IndexedRecord> indexedRecords = () -> {
try {
- HoodieFileReaderFactory fileReaderFactory = HoodieIOFactory.getIOFactory(table.getStorageConf())
+ HoodieFileReaderFactory fileReaderFactory = HoodieIOFactory.getIOFactory(table.getStorage())
.getReaderFactory(table.getConfig().getRecordMerger().getRecordType());
HoodieAvroFileReader fileReader = (HoodieAvroFileReader) fileReaderFactory.getFileReader(
table.getConfig(), new StoragePath(clusteringOp.getDataFilePath()));
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfiles.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfiles.java
index 1536dae35ba..b2c7d56f624 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfiles.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfiles.java
@@ -31,6 +31,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.StreamerUtil;
import org.apache.flink.core.fs.Path;
@@ -148,9 +149,9 @@ public class WriteProfiles {
) {
switch (tableType) {
case COPY_ON_WRITE:
- return metadata.getFileIdToInfo(HadoopFSUtils.getStorageConf(hadoopConf), basePath);
+ return metadata.getFileIdToInfo(basePath);
case MERGE_ON_READ:
- return metadata.getFullPathToInfo(HadoopFSUtils.getStorageConf(hadoopConf), basePath);
+ return metadata.getFullPathToInfo(new HoodieHadoopStorage(basePath, hadoopConf), basePath);
default:
throw new AssertionError();
}
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java
index 3361a58e4e9..408823f6958 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java
@@ -24,13 +24,15 @@ import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.configuration.HadoopConfigurations;
+import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.index.bucket.BucketIdentifier;
import org.apache.hudi.source.prune.DataPruner;
import org.apache.hudi.source.prune.PartitionPruners;
import org.apache.hudi.source.prune.PrimaryKeyPruners;
import org.apache.hudi.source.stats.ColumnStatsIndices;
-import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.DataTypeUtils;
import org.apache.hudi.util.StreamerUtil;
@@ -153,7 +155,8 @@ public class FileIndex implements Serializable {
String[] partitions =
getOrBuildPartitionPaths().stream().map(p -> fullPartitionPath(path, p)).toArray(String[]::new);
List<StoragePathInfo> allFiles = FSUtils.getFilesInPartitions(
- new HoodieFlinkEngineContext(hadoopConf), metadataConfig, path.toString(), partitions)
+ new HoodieFlinkEngineContext(hadoopConf),
+ new HoodieHadoopStorage(path, HadoopFSUtils.getStorageConf(hadoopConf)), metadataConfig, path.toString(), partitions)
.values().stream()
.flatMap(Collection::stream)
.collect(Collectors.toList());
@@ -281,7 +284,8 @@ public class FileIndex implements Serializable {
return this.partitionPaths;
}
List<String> allPartitionPaths = this.tableExists
- ? FSUtils.getAllPartitionPaths(new HoodieFlinkEngineContext(hadoopConf), metadataConfig, path.toString())
+ ? FSUtils.getAllPartitionPaths(new HoodieFlinkEngineContext(hadoopConf),
+ new HoodieHadoopStorage(path, HadoopFSUtils.getStorageConf(hadoopConf)), metadataConfig, path.toString())
: Collections.emptyList();
if (this.partitionPruner == null) {
this.partitionPaths = allPartitionPaths;
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java
index 7032f299368..674e58dce6b 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java
@@ -31,8 +31,10 @@ import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.metadata.HoodieMetadataPayload;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.AvroSchemaConverter;
import org.apache.hudi.util.AvroToRowDataConverters;
+import org.apache.hudi.util.FlinkClientUtil;
import org.apache.hudi.util.RowDataProjection;
import org.apache.avro.generic.GenericRecord;
@@ -285,7 +287,7 @@ public class ColumnStatsIndices {
// - Fetching the records from CSI by key-prefixes (encoded column names)
// - Deserializing fetched records into [[RowData]]s
HoodieTableMetadata metadataTable = HoodieTableMetadata.create(
- HoodieFlinkEngineContext.DEFAULT,
+ HoodieFlinkEngineContext.DEFAULT, new HoodieHadoopStorage(basePath, FlinkClientUtil.getHadoopConf()),
metadataConfig, basePath);
// TODO encoding should be done internally w/in HoodieBackedTableMetadata
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/InternalSchemaManager.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/InternalSchemaManager.java
index 9203e6dd11b..c5726b9d844 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/InternalSchemaManager.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/InternalSchemaManager.java
@@ -27,13 +27,13 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.configuration.HadoopConfigurations;
import org.apache.hudi.configuration.OptionsResolver;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.internal.schema.InternalSchema;
import org.apache.hudi.internal.schema.Type;
import org.apache.hudi.internal.schema.Types;
import org.apache.hudi.internal.schema.action.InternalSchemaMerger;
import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter;
import org.apache.hudi.internal.schema.utils.InternalSchemaUtils;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.AvroSchemaConverter;
import org.apache.flink.configuration.Configuration;
@@ -111,7 +111,9 @@ public class InternalSchemaManager implements Serializable {
}
long commitInstantTime = Long.parseLong(FSUtils.getCommitTime(fileName));
InternalSchema fileSchema = InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, HadoopFSUtils.getStorageConf(getHadoopConf()), validCommits);
+ commitInstantTime, tablePath,
+ new HoodieHadoopStorage(tablePath, getHadoopConf()),
+ validCommits);
if (querySchema.equals(fileSchema)) {
return InternalSchema.getEmptyInternalSchema();
}
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java
index 57966b4bdbf..8cd8be8b5cc 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java
@@ -35,12 +35,11 @@ import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.configuration.OptionsResolver;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.internal.schema.InternalSchema;
import org.apache.hudi.source.ExpressionPredicates.Predicate;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.table.format.FormatUtils;
import org.apache.hudi.table.format.InternalSchemaManager;
import org.apache.hudi.table.format.mor.MergeOnReadInputFormat;
@@ -336,7 +335,7 @@ public class CdcInputFormat extends MergeOnReadInputFormat {
this.recordBuilder = new GenericRecordBuilder(requiredSchema);
this.avroToRowDataConverter = AvroToRowDataConverters.createRowConverter(tableState.getRequiredRowType());
StoragePath hadoopTablePath = new StoragePath(tablePath);
- HoodieStorage storage = HoodieStorageUtils.getStorage(hadoopTablePath, HadoopFSUtils.getStorageConf(hadoopConf));
+ HoodieStorage storage = new HoodieHadoopStorage(tablePath, hadoopConf);
HoodieLogFile[] cdcLogFiles = fileSplit.getCdcFiles().stream().map(cdcFile -> {
try {
return new HoodieLogFile(
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
index 07ff489e0cb..c52371ae0a3 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
@@ -244,7 +244,7 @@ public class TestStreamWriteOperatorCoordinator {
String basePath = tempFile.getAbsolutePath();
HoodieStorage storage =
- coordinator.getWriteClient().getHoodieTable().getMetaClient().getStorage();
+ coordinator.getWriteClient().getHoodieTable().getStorage();
assertTrue(HoodieHeartbeatClient.heartbeatExists(storage, basePath, instant),
"Heartbeat is existed");
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java
index fcc6b679f93..25fc9564cec 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java
@@ -420,7 +420,7 @@ public class ITTestHoodieFlinkCompactor {
Set<Pair<String, String>> fileIdCommitTimeSet = new HashSet<>();
HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(conf);
HoodieStorage storage = metaClient.getStorage();
- FSUtils.getAllPartitionPaths(HoodieFlinkEngineContext.DEFAULT, metaClient.getBasePath(), false).forEach(
+ FSUtils.getAllPartitionPaths(HoodieFlinkEngineContext.DEFAULT, metaClient.getStorage(), metaClient.getBasePath(), false).forEach(
partition -> {
try {
storage.listDirectEntries(FSUtils.constructAbsolutePath(metaClient.getBasePathV2(), partition))
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
index 2e3f546debe..3941c28e570 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hadoop.conf.Configuration;
import org.slf4j.Logger;
@@ -147,7 +148,7 @@ public class DFSPropertiesConfiguration extends PropertiesConfig {
throw new IllegalStateException("Loop detected; file " + filePath + " already referenced");
}
- HoodieStorage storage = HoodieStorageUtils.getStorage(
+ HoodieStorage storage = new HoodieHadoopStorage(
filePath,
HadoopFSUtils.getStorageConf(Option.ofNullable(hadoopConfig).orElseGet(Configuration::new))
);
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
index e00dfd53f02..32d3b5b98f2 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
@@ -148,7 +148,7 @@ public class HoodieLogFormatWriter implements HoodieLogFormat.Writer {
// bytes for header
byte[] headerBytes = HoodieLogBlock.getLogMetadataBytes(block.getLogBlockHeader());
// content bytes
- byte[] content = block.getContentBytes(storage.getConf());
+ byte[] content = block.getContentBytes(storage);
// bytes for footer
byte[] footerBytes = HoodieLogBlock.getLogMetadataBytes(block.getLogBlockFooter());
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
index b746a92250e..28b0dc2c42c 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
@@ -34,7 +34,6 @@ import org.apache.hudi.io.storage.HoodieHBaseKVComparator;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -85,56 +84,56 @@ public class HFileUtils extends FileFormatUtils {
}
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath) {
throw new UnsupportedOperationException("HFileUtils does not support readAvroRecords");
}
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath, Schema schema) {
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath, Schema schema) {
throw new UnsupportedOperationException("HFileUtils does not support readAvroRecords");
}
@Override
- public Map<String, String> readFooter(StorageConfiguration<?> configuration, boolean required, StoragePath filePath, String... footerNames) {
+ public Map<String, String> readFooter(HoodieStorage storage, boolean required, StoragePath filePath, String... footerNames) {
throw new UnsupportedOperationException("HFileUtils does not support readFooter");
}
@Override
- public long getRowCount(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public long getRowCount(HoodieStorage storage, StoragePath filePath) {
throw new UnsupportedOperationException("HFileUtils does not support getRowCount");
}
@Override
- public Set<Pair<String, Long>> filterRowKeys(StorageConfiguration<?> configuration, StoragePath filePath, Set<String> filter) {
+ public Set<Pair<String, Long>> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set<String> filter) {
throw new UnsupportedOperationException("HFileUtils does not support filterRowKeys");
}
@Override
- public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath) {
throw new UnsupportedOperationException("HFileUtils does not support fetchRecordKeysWithPositions");
}
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
throw new UnsupportedOperationException("HFileUtils does not support getHoodieKeyIterator");
}
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath) {
throw new UnsupportedOperationException("HFileUtils does not support getHoodieKeyIterator");
}
@Override
- public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
throw new UnsupportedOperationException("HFileUtils does not support fetchRecordKeysWithPositions");
}
@Override
- public Schema readAvroSchema(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public Schema readAvroSchema(HoodieStorage storage, StoragePath filePath) {
LOG.info("Reading schema from {}", filePath);
try (HoodieFileReader fileReader =
- HoodieIOFactory.getIOFactory(configuration)
+ HoodieIOFactory.getIOFactory(storage)
.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(
ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER,
@@ -146,7 +145,7 @@ public class HFileUtils extends FileFormatUtils {
}
@Override
- public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(StorageConfiguration<?> storageConf, StoragePath filePath, List<String> columnList) {
+ public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(HoodieStorage storage, StoragePath filePath, List<String> columnList) {
throw new UnsupportedOperationException(
"Reading column statistics from metadata is not supported for HFile format yet");
}
@@ -162,7 +161,7 @@ public class HFileUtils extends FileFormatUtils {
}
@Override
- public byte[] serializeRecordsToLogBlock(StorageConfiguration<?> storageConf,
+ public byte[] serializeRecordsToLogBlock(HoodieStorage storage,
List<HoodieRecord> records,
Schema writerSchema,
Schema readerSchema,
@@ -175,7 +174,7 @@ public class HFileUtils extends FileFormatUtils {
.withCellComparator(new HoodieHBaseKVComparator())
.build();
- Configuration conf = storageConf.unwrapAs(Configuration.class);
+ Configuration conf = storage.getConf().unwrapAs(Configuration.class);
CacheConfig cacheConfig = new CacheConfig(conf);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
FSDataOutputStream ostream = new FSDataOutputStream(baos, null);
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
index 0a8734790c7..d58acd3807b 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
@@ -32,8 +32,6 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.hadoop.OrcReaderIterator;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -74,14 +72,14 @@ public class OrcUtils extends FileFormatUtils {
/**
* Provides a closable iterator for reading the given ORC file.
*
- * @param configuration configuration to build storage object
- * @param filePath The ORC file path
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath The ORC file path
* @return {@link ClosableIterator} of {@link HoodieKey}s for reading the ORC file
*/
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath) {
try {
- Configuration conf = configuration.unwrapCopyAs(Configuration.class);
+ Configuration conf = storage.getConf().unwrapCopyAs(Configuration.class);
conf.addResource(HadoopFSUtils.getFs(filePath.toString(), conf).getConf());
Reader reader = OrcFile.createReader(convertToHadoopPath(filePath), OrcFile.readerOptions(conf));
@@ -113,14 +111,14 @@ public class OrcUtils extends FileFormatUtils {
/**
* Fetch {@link HoodieKey}s from the given ORC file.
*
- * @param configuration configuration to build storage object
- * @param filePath The ORC file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath The ORC file path.
* @return {@link List} of {@link HoodieKey}s fetched from the ORC file
*/
@Override
- public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath) {
try {
- if (!HoodieStorageUtils.getStorage(filePath, configuration).exists(filePath)) {
+ if (!storage.exists(filePath)) {
return Collections.emptyList();
}
} catch (IOException e) {
@@ -128,7 +126,7 @@ public class OrcUtils extends FileFormatUtils {
}
List<Pair<HoodieKey, Long>> hoodieKeysAndPositions = new ArrayList<>();
long position = 0;
- try (ClosableIterator<HoodieKey> iterator = getHoodieKeyIterator(configuration, filePath, Option.empty())) {
+ try (ClosableIterator<HoodieKey> iterator = getHoodieKeyIterator(storage, filePath, Option.empty())) {
while (iterator.hasNext()) {
hoodieKeysAndPositions.add(Pair.of(iterator.next(), position));
position++;
@@ -138,12 +136,12 @@ public class OrcUtils extends FileFormatUtils {
}
@Override
- public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
throw new UnsupportedOperationException("Custom key generator is not supported yet");
}
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
throw new UnsupportedOperationException("Custom key generator is not supported yet");
}
@@ -151,28 +149,28 @@ public class OrcUtils extends FileFormatUtils {
* NOTE: This literally reads the entire file contents, thus should be used with caution.
*/
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath) {
Schema avroSchema;
try (Reader reader = OrcFile.createReader(
- convertToHadoopPath(filePath), OrcFile.readerOptions(configuration.unwrapAs(Configuration.class)))) {
+ convertToHadoopPath(filePath), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) {
avroSchema = AvroOrcUtils.createAvroSchema(reader.getSchema());
} catch (IOException io) {
throw new HoodieIOException("Unable to read Avro records from an ORC file:" + filePath, io);
}
- return readAvroRecords(configuration, filePath, avroSchema);
+ return readAvroRecords(storage, filePath, avroSchema);
}
/**
* NOTE: This literally reads the entire file contents, thus should be used with caution.
*/
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath, Schema avroSchema) {
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath, Schema avroSchema) {
List<GenericRecord> records = new ArrayList<>();
try (Reader reader = OrcFile.createReader(
- convertToHadoopPath(filePath), OrcFile.readerOptions(configuration.unwrapAs(Configuration.class)))) {
+ convertToHadoopPath(filePath), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) {
TypeDescription orcSchema = reader.getSchema();
try (RecordReader recordReader = reader.rows(
- new Options(configuration.unwrapAs(Configuration.class)).schema(orcSchema))) {
+ new Options(storage.getConf().unwrapAs(Configuration.class)).schema(orcSchema))) {
OrcReaderIterator<GenericRecord> iterator = new OrcReaderIterator<>(recordReader, avroSchema, orcSchema);
while (iterator.hasNext()) {
GenericRecord record = iterator.next();
@@ -189,19 +187,19 @@ public class OrcUtils extends FileFormatUtils {
* Read the rowKey list matching the given filter, from the given ORC file. If the filter is empty, then this will
* return all the rowkeys.
*
- * @param conf configuration to build storage object.
+ * @param storage {@link HoodieStorage} instance.
* @param filePath The ORC file path.
* @param filter record keys filter
* @return Set Set of pairs of row key and position matching candidateRecordKeys
*/
@Override
- public Set<Pair<String, Long>> filterRowKeys(StorageConfiguration<?> conf, StoragePath filePath, Set<String> filter)
+ public Set<Pair<String, Long>> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set<String> filter)
throws HoodieIOException {
long rowPosition = 0;
try (Reader reader = OrcFile.createReader(
- convertToHadoopPath(filePath), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)))) {
+ convertToHadoopPath(filePath), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) {
TypeDescription schema = reader.getSchema();
- try (RecordReader recordReader = reader.rows(new Options(conf.unwrapAs(Configuration.class)).schema(schema))) {
+ try (RecordReader recordReader = reader.rows(new Options(storage.getConf().unwrapAs(Configuration.class)).schema(schema))) {
Set<Pair<String, Long>> filteredRowKeys = new HashSet<>();
List<String> fieldNames = schema.getFieldNames();
VectorizedRowBatch batch = schema.createRowBatch();
@@ -235,10 +233,10 @@ public class OrcUtils extends FileFormatUtils {
}
@Override
- public Map<String, String> readFooter(StorageConfiguration<?> conf, boolean required,
+ public Map<String, String> readFooter(HoodieStorage storage, boolean required,
StoragePath filePath, String... footerNames) {
try (Reader reader = OrcFile.createReader(
- convertToHadoopPath(filePath), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)))) {
+ convertToHadoopPath(filePath), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) {
Map<String, String> footerVals = new HashMap<>();
List<UserMetadataItem> metadataItemList = reader.getFileTail().getFooter().getMetadataList();
Map<String, String> metadata = metadataItemList.stream().collect(Collectors.toMap(
@@ -259,9 +257,9 @@ public class OrcUtils extends FileFormatUtils {
}
@Override
- public Schema readAvroSchema(StorageConfiguration<?> conf, StoragePath filePath) {
+ public Schema readAvroSchema(HoodieStorage storage, StoragePath filePath) {
try (Reader reader = OrcFile.createReader(
- convertToHadoopPath(filePath), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)))) {
+ convertToHadoopPath(filePath), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) {
if (reader.hasMetadataValue("orc.avro.schema")) {
ByteBuffer metadataValue = reader.getMetadataValue("orc.avro.schema");
byte[] bytes = toBytes(metadataValue);
@@ -276,7 +274,7 @@ public class OrcUtils extends FileFormatUtils {
}
@Override
- public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(StorageConfiguration<?> storageConf, StoragePath filePath, List<String> columnList) {
+ public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(HoodieStorage storage, StoragePath filePath, List<String> columnList) {
throw new UnsupportedOperationException(
"Reading column statistics from metadata is not supported for ORC format yet");
}
@@ -287,9 +285,9 @@ public class OrcUtils extends FileFormatUtils {
}
@Override
- public long getRowCount(StorageConfiguration<?> conf, StoragePath filePath) {
+ public long getRowCount(HoodieStorage storage, StoragePath filePath) {
try (Reader reader = OrcFile.createReader(
- convertToHadoopPath(filePath), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)))) {
+ convertToHadoopPath(filePath), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) {
return reader.getNumberOfRows();
} catch (IOException io) {
throw new HoodieIOException("Unable to get row count for ORC file:" + filePath, io);
@@ -312,7 +310,7 @@ public class OrcUtils extends FileFormatUtils {
}
@Override
- public byte[] serializeRecordsToLogBlock(StorageConfiguration<?> storageConf,
+ public byte[] serializeRecordsToLogBlock(HoodieStorage storage,
List<HoodieRecord> records,
Schema writerSchema,
Schema readerSchema, String keyFieldName,
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
index 76b3aaf058f..316413d3195 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
@@ -34,8 +34,6 @@ import org.apache.hudi.io.storage.HoodieFileWriter;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -82,6 +80,7 @@ import java.util.stream.Stream;
import static org.apache.hudi.common.config.HoodieStorageConfig.PARQUET_BLOCK_SIZE;
import static org.apache.hudi.common.config.HoodieStorageConfig.PARQUET_MAX_FILE_SIZE;
import static org.apache.hudi.common.config.HoodieStorageConfig.PARQUET_PAGE_SIZE;
+import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath;
/**
* Utility functions involving with parquet.
@@ -94,23 +93,23 @@ public class ParquetUtils extends FileFormatUtils {
* Read the rowKey list matching the given filter, from the given parquet file. If the filter is empty, then this will
* return all the rowkeys and corresponding positions.
*
- * @param configuration configuration to build storage object
- * @param filePath The parquet file path.
- * @param filter record keys filter
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath The parquet file path.
+ * @param filter record keys filter
* @return Set Set of pairs of row key and position matching candidateRecordKeys
*/
@Override
- public Set<Pair<String, Long>> filterRowKeys(StorageConfiguration<?> configuration, StoragePath filePath, Set<String> filter) {
- return filterParquetRowKeys(configuration, new Path(filePath.toUri()), filter, HoodieAvroUtils.getRecordKeySchema());
+ public Set<Pair<String, Long>> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set<String> filter) {
+ return filterParquetRowKeys(storage, new Path(filePath.toUri()), filter, HoodieAvroUtils.getRecordKeySchema());
}
- public static ParquetMetadata readMetadata(StorageConfiguration<?> conf, StoragePath parquetFilePath) {
+ public static ParquetMetadata readMetadata(HoodieStorage storage, StoragePath parquetFilePath) {
Path parquetFileHadoopPath = new Path(parquetFilePath.toUri());
ParquetMetadata footer;
try {
// TODO(vc): Should we use the parallel reading version here?
- footer = ParquetFileReader.readFooter(HoodieStorageUtils.getStorage(
- parquetFileHadoopPath.toString(), conf).getConf().unwrapAs(Configuration.class), parquetFileHadoopPath);
+ footer = ParquetFileReader.readFooter(storage.newInstance(
+ parquetFilePath, storage.getConf()).getConf().unwrapAs(Configuration.class), parquetFileHadoopPath);
} catch (IOException e) {
throw new HoodieIOException("Failed to read footer for parquet " + parquetFileHadoopPath, e);
}
@@ -121,21 +120,21 @@ public class ParquetUtils extends FileFormatUtils {
* Read the rowKey list matching the given filter, from the given parquet file. If the filter is empty, then this will
* return all the rowkeys.
*
- * @param filePath The parquet file path.
- * @param configuration configuration to build storage object
- * @param filter record keys filter
- * @param readSchema schema of columns to be read
- * @return Set Set of pairs of row key and position matching candidateRecordKeys
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath The parquet file path.
+ * @param filter record keys filter
+ * @param readSchema schema of columns to be read
+ * @return Set of pairs of row key and position matching candidateRecordKeys
*/
- private static Set<Pair<String, Long>> filterParquetRowKeys(StorageConfiguration<?> configuration,
+ private static Set<Pair<String, Long>> filterParquetRowKeys(HoodieStorage storage,
Path filePath, Set<String> filter,
Schema readSchema) {
Option<RecordKeysFilterFunction> filterFunction = Option.empty();
if (filter != null && !filter.isEmpty()) {
filterFunction = Option.of(new RecordKeysFilterFunction(filter));
}
- Configuration conf = configuration.unwrapCopyAs(Configuration.class);
- conf.addResource(HoodieStorageUtils.getStorage(filePath.toString(), configuration).getConf().unwrapAs(Configuration.class));
+ Configuration conf = storage.getConf().unwrapCopyAs(Configuration.class);
+ conf.addResource(storage.newInstance(convertToStoragePath(filePath), storage.getConf()).getConf().unwrapAs(Configuration.class));
AvroReadSupport.setAvroReadSchema(conf, readSchema);
AvroReadSupport.setRequestedProjection(conf, readSchema);
Set<Pair<String, Long>> rowKeys = new HashSet<>();
@@ -171,39 +170,39 @@ public class ParquetUtils extends FileFormatUtils {
/**
* Fetch {@link HoodieKey}s with row positions from the given parquet file.
*
- * @param configuration configuration to build storage object
- * @param filePath The parquet file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath The parquet file path.
* @return {@link List} of pairs of {@link HoodieKey} and row position fetched from the parquet file
*/
@Override
- public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(StorageConfiguration<?> configuration, StoragePath filePath) {
- return fetchRecordKeysWithPositions(configuration, filePath, Option.empty());
+ public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath) {
+ return fetchRecordKeysWithPositions(storage, filePath, Option.empty());
}
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath) {
- return getHoodieKeyIterator(configuration, filePath, Option.empty());
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath) {
+ return getHoodieKeyIterator(storage, filePath, Option.empty());
}
/**
* Returns a closable iterator for reading the given parquet file.
*
- * @param configuration configuration to build storage object
+ * @param storage {@link HoodieStorage} instance.
* @param filePath The parquet file path
* @param keyGeneratorOpt instance of KeyGenerator
* @return {@link ClosableIterator} of {@link HoodieKey}s for reading the parquet file
*/
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
try {
- Configuration conf = configuration.unwrapCopyAs(Configuration.class);
- conf.addResource(HoodieStorageUtils.getStorage(filePath.toString(), configuration).getConf().unwrapAs(Configuration.class));
+ Configuration conf = storage.getConf().unwrapCopyAs(Configuration.class);
+ conf.addResource(storage.newInstance(filePath, storage.getConf()).getConf().unwrapAs(Configuration.class));
Schema readSchema = keyGeneratorOpt
.map(keyGenerator -> {
List<String> fields = new ArrayList<>();
fields.addAll(keyGenerator.getRecordKeyFieldNames());
fields.addAll(keyGenerator.getPartitionPathFields());
- return HoodieAvroUtils.getSchemaForFields(readAvroSchema(configuration, filePath), fields);
+ return HoodieAvroUtils.getSchemaForFields(readAvroSchema(storage, filePath), fields);
})
.orElse(HoodieAvroUtils.getRecordKeyPartitionPathSchema());
AvroReadSupport.setAvroReadSchema(conf, readSchema);
@@ -219,16 +218,16 @@ public class ParquetUtils extends FileFormatUtils {
/**
* Fetch {@link HoodieKey}s with row positions from the given parquet file.
*
- * @param configuration configuration to build storage object
+ * @param storage {@link HoodieStorage} instance.
* @param filePath The parquet file path.
* @param keyGeneratorOpt instance of KeyGenerator.
* @return {@link List} of pairs of {@link HoodieKey} and row position fetched from the parquet file
*/
@Override
- public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
List<Pair<HoodieKey, Long>> hoodieKeysAndPositions = new ArrayList<>();
long position = 0;
- try (ClosableIterator<HoodieKey> iterator = getHoodieKeyIterator(configuration, filePath, keyGeneratorOpt)) {
+ try (ClosableIterator<HoodieKey> iterator = getHoodieKeyIterator(storage, filePath, keyGeneratorOpt)) {
while (iterator.hasNext()) {
hoodieKeysAndPositions.add(Pair.of(iterator.next(), position));
position++;
@@ -240,15 +239,15 @@ public class ParquetUtils extends FileFormatUtils {
/**
* Get the schema of the given parquet file.
*/
- public MessageType readSchema(StorageConfiguration<?> configuration, StoragePath parquetFilePath) {
- return readMetadata(configuration, parquetFilePath).getFileMetaData().getSchema();
+ public MessageType readSchema(HoodieStorage storage, StoragePath parquetFilePath) {
+ return readMetadata(storage, parquetFilePath).getFileMetaData().getSchema();
}
@Override
- public Map<String, String> readFooter(StorageConfiguration<?> configuration, boolean required,
+ public Map<String, String> readFooter(HoodieStorage storage, boolean required,
StoragePath filePath, String... footerNames) {
Map<String, String> footerVals = new HashMap<>();
- ParquetMetadata footer = readMetadata(configuration, filePath);
+ ParquetMetadata footer = readMetadata(storage, filePath);
Map<String, String> metadata = footer.getFileMetaData().getKeyValueMetaData();
for (String footerName : footerNames) {
if (metadata.containsKey(footerName)) {
@@ -262,16 +261,16 @@ public class ParquetUtils extends FileFormatUtils {
}
@Override
- public Schema readAvroSchema(StorageConfiguration<?> conf, StoragePath filePath) {
- MessageType parquetSchema = readSchema(conf, filePath);
- return new AvroSchemaConverter(conf.unwrapAs(Configuration.class)).convert(parquetSchema);
+ public Schema readAvroSchema(HoodieStorage storage, StoragePath filePath) {
+ MessageType parquetSchema = readSchema(storage, filePath);
+ return new AvroSchemaConverter(storage.getConf().unwrapAs(Configuration.class)).convert(parquetSchema);
}
@Override
- public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(StorageConfiguration<?> storageConf,
+ public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(HoodieStorage storage,
StoragePath filePath,
List<String> columnList) {
- ParquetMetadata metadata = readMetadata(storageConf, filePath);
+ ParquetMetadata metadata = readMetadata(storage, filePath);
// NOTE: This collector has to have fully specialized generic type params since
// Java 1.8 struggles to infer them
@@ -325,10 +324,10 @@ public class ParquetUtils extends FileFormatUtils {
* NOTE: This literally reads the entire file contents, thus should be used with caution.
*/
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath) {
List<GenericRecord> records = new ArrayList<>();
try (ParquetReader reader = AvroParquetReader.builder(new Path(filePath.toUri()))
- .withConf(configuration.unwrapAs(Configuration.class)).build()) {
+ .withConf(storage.getConf().unwrapAs(Configuration.class)).build()) {
Object obj = reader.read();
while (obj != null) {
if (obj instanceof GenericRecord) {
@@ -344,22 +343,22 @@ public class ParquetUtils extends FileFormatUtils {
}
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath, Schema schema) {
- AvroReadSupport.setAvroReadSchema(configuration.unwrapAs(Configuration.class), schema);
- return readAvroRecords(configuration, filePath);
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath, Schema schema) {
+ AvroReadSupport.setAvroReadSchema(storage.getConf().unwrapAs(Configuration.class), schema);
+ return readAvroRecords(storage, filePath);
}
/**
* Returns the number of records in the parquet file.
*
- * @param conf Configuration
+ * @param storage {@link HoodieStorage} instance.
* @param filePath path of the file
*/
@Override
- public long getRowCount(StorageConfiguration<?> conf, StoragePath filePath) {
+ public long getRowCount(HoodieStorage storage, StoragePath filePath) {
ParquetMetadata footer;
long rowCount = 0;
- footer = readMetadata(conf, filePath);
+ footer = readMetadata(storage, filePath);
for (BlockMetaData b : footer.getBlocks()) {
rowCount += b.getRowCount();
}
@@ -383,7 +382,7 @@ public class ParquetUtils extends FileFormatUtils {
}
@Override
- public byte[] serializeRecordsToLogBlock(StorageConfiguration<?> storageConf,
+ public byte[] serializeRecordsToLogBlock(HoodieStorage storage,
List<HoodieRecord> records,
Schema writerSchema,
Schema readerSchema,
@@ -401,7 +400,7 @@ public class ParquetUtils extends FileFormatUtils {
config.setValue(PARQUET_MAX_FILE_SIZE.key(), String.valueOf(1024 * 1024 * 1024));
HoodieRecord.HoodieRecordType recordType = records.iterator().next().getRecordType();
try (HoodieFileWriter parquetWriter = HoodieFileWriterFactory.getFileWriter(
- HoodieFileFormat.PARQUET, outputStream, storageConf, config, writerSchema, recordType)) {
+ HoodieFileFormat.PARQUET, outputStream, storage, config, writerSchema, recordType)) {
for (HoodieRecord<?> record : records) {
String recordKey = record.getRecordKey(readerSchema, keyFieldName);
parquetWriter.write(recordKey, record, writerSchema);
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java
index d3a340adfbb..1c455240be1 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java
@@ -26,7 +26,6 @@ import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.io.storage.HoodieNativeAvroHFileReader;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -35,13 +34,13 @@ import java.io.IOException;
public class HoodieAvroFileReaderFactory extends HoodieFileReaderFactory {
- public HoodieAvroFileReaderFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieAvroFileReaderFactory(HoodieStorage storage) {
+ super(storage);
}
@Override
protected HoodieFileReader newParquetFileReader(StoragePath path) {
- return new HoodieAvroParquetReader(storageConf, path);
+ return new HoodieAvroParquetReader(storage, path);
}
@Override
@@ -49,12 +48,12 @@ public class HoodieAvroFileReaderFactory extends HoodieFileReaderFactory {
StoragePath path,
Option<Schema> schemaOption) throws IOException {
if (isUseNativeHFileReaderEnabled(hoodieConfig)) {
- return new HoodieNativeAvroHFileReader(storageConf, path, schemaOption);
+ return new HoodieNativeAvroHFileReader(storage, path, schemaOption);
}
if (schemaOption.isPresent()) {
- return new HoodieHBaseAvroHFileReader(storageConf, path, schemaOption);
+ return new HoodieHBaseAvroHFileReader(storage.getConf(), path, schemaOption);
}
- return new HoodieHBaseAvroHFileReader(storageConf, path);
+ return new HoodieHBaseAvroHFileReader(storage.getConf(), path);
}
@Override
@@ -64,14 +63,14 @@ public class HoodieAvroFileReaderFactory extends HoodieFileReaderFactory {
byte[] content,
Option<Schema> schemaOption) throws IOException {
if (isUseNativeHFileReaderEnabled(hoodieConfig)) {
- return new HoodieNativeAvroHFileReader(storageConf, content, schemaOption);
+ return new HoodieNativeAvroHFileReader(this.storage, content, schemaOption);
}
- return new HoodieHBaseAvroHFileReader(storageConf, path, storage, content, schemaOption);
+ return new HoodieHBaseAvroHFileReader(this.storage.getConf(), path, storage, content, schemaOption);
}
@Override
protected HoodieFileReader newOrcFileReader(StoragePath path) {
- return new HoodieAvroOrcReader(storageConf, path);
+ return new HoodieAvroOrcReader(storage, path);
}
@Override
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java
index 0ce60074c2d..aaf583afefb 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java
@@ -32,7 +32,7 @@ import org.apache.hudi.io.storage.HoodieFileWriter;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.io.storage.HoodieOrcConfig;
import org.apache.hudi.io.storage.HoodieParquetConfig;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -55,8 +55,8 @@ import static org.apache.hudi.io.hadoop.HoodieHFileConfig.PREFETCH_ON_OPEN;
public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
- public HoodieAvroFileWriterFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieAvroFileWriterFactory(HoodieStorage storage) {
+ super(storage);
}
@Override
@@ -76,7 +76,7 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
config.getIntOrDefault(HoodieStorageConfig.PARQUET_BLOCK_SIZE),
config.getIntOrDefault(HoodieStorageConfig.PARQUET_PAGE_SIZE),
config.getLongOrDefault(HoodieStorageConfig.PARQUET_MAX_FILE_SIZE),
- storageConf, config.getDoubleOrDefault(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION),
+ storage.getConf(), config.getDoubleOrDefault(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION),
config.getBooleanOrDefault(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED));
return new HoodieAvroParquetWriter(path, parquetConfig, instantTime, taskContextSupplier, populateMetaFields);
}
@@ -89,7 +89,7 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
config.getInt(HoodieStorageConfig.PARQUET_BLOCK_SIZE),
config.getInt(HoodieStorageConfig.PARQUET_PAGE_SIZE),
config.getLong(HoodieStorageConfig.PARQUET_MAX_FILE_SIZE), // todo: 1024*1024*1024
- storageConf, config.getDouble(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION),
+ storage.getConf(), config.getDouble(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION),
config.getBoolean(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED));
return new HoodieParquetStreamWriter(new FSDataOutputStream(outputStream, null), parquetConfig);
}
@@ -98,7 +98,7 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
String instantTime, StoragePath path, HoodieConfig config, Schema schema,
TaskContextSupplier taskContextSupplier) throws IOException {
BloomFilter filter = createBloomFilter(config);
- HoodieHFileConfig hfileConfig = new HoodieHFileConfig(storageConf.unwrapAs(Configuration.class),
+ HoodieHFileConfig hfileConfig = new HoodieHFileConfig(storage.getConf().unwrapAs(Configuration.class),
Compression.Algorithm.valueOf(
config.getString(HoodieStorageConfig.HFILE_COMPRESSION_ALGORITHM_NAME)),
config.getInt(HoodieStorageConfig.HFILE_BLOCK_SIZE),
@@ -112,7 +112,7 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
String instantTime, StoragePath path, HoodieConfig config, Schema schema,
TaskContextSupplier taskContextSupplier) throws IOException {
BloomFilter filter = createBloomFilter(config);
- HoodieOrcConfig orcConfig = new HoodieOrcConfig(storageConf,
+ HoodieOrcConfig orcConfig = new HoodieOrcConfig(storage.getConf(),
CompressionKind.valueOf(config.getString(HoodieStorageConfig.ORC_COMPRESSION_CODEC_NAME)),
config.getInt(HoodieStorageConfig.ORC_STRIPE_SIZE),
config.getInt(HoodieStorageConfig.ORC_BLOCK_SIZE),
@@ -126,6 +126,6 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
return (HoodieAvroWriteSupport) ReflectionUtils.loadClass(
config.getStringOrDefault(HoodieStorageConfig.HOODIE_AVRO_WRITE_SUPPORT_CLASS),
new Class<?>[] {MessageType.class, Schema.class, Option.class, Properties.class},
- new AvroSchemaConverter(storageConf.unwrapAs(Configuration.class)).convert(schema), schema, filter, config.getProps());
+ new AvroSchemaConverter(storage.getConf().unwrapAs(Configuration.class)).convert(schema), schema, filter, config.getProps());
}
}
\ No newline at end of file
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
index 424be01b044..2c603839dc8 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
@@ -28,7 +28,7 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieFileReader;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -52,28 +52,28 @@ import java.util.Set;
public class HoodieAvroOrcReader extends HoodieAvroFileReader {
private final StoragePath path;
- private final StorageConfiguration<?> conf;
+ private final HoodieStorage storage;
private final FileFormatUtils orcUtils;
- public HoodieAvroOrcReader(StorageConfiguration<?> configuration, StoragePath path) {
- this.conf = configuration;
+ public HoodieAvroOrcReader(HoodieStorage storage, StoragePath path) {
+ this.storage = storage;
this.path = path;
this.orcUtils = FileFormatUtils.getInstance(HoodieFileFormat.ORC);
}
@Override
public String[] readMinMaxRecordKeys() {
- return orcUtils.readMinMaxRecordKeys(conf, path);
+ return orcUtils.readMinMaxRecordKeys(storage, path);
}
@Override
public BloomFilter readBloomFilter() {
- return orcUtils.readBloomFilterFromMetadata(conf, path);
+ return orcUtils.readBloomFilterFromMetadata(storage, path);
}
@Override
public Set<Pair<String, Long>> filterRowKeys(Set candidateRowKeys) {
- return orcUtils.filterRowKeys(conf, path, candidateRowKeys);
+ return orcUtils.filterRowKeys(storage, path, candidateRowKeys);
}
@Override
@@ -82,7 +82,7 @@ public class HoodieAvroOrcReader extends HoodieAvroFileReader {
throw new UnsupportedOperationException("Schema projections are not supported in HFile reader");
}
- Configuration hadoopConf = conf.unwrapAs(Configuration.class);
+ Configuration hadoopConf = storage.getConf().unwrapAs(Configuration.class);
try (Reader reader = OrcFile.createReader(new Path(path.toUri()), OrcFile.readerOptions(hadoopConf))) {
TypeDescription orcSchema = AvroOrcUtils.createOrcSchema(readerSchema);
RecordReader recordReader = reader.rows(new Options(hadoopConf).schema(orcSchema));
@@ -94,7 +94,7 @@ public class HoodieAvroOrcReader extends HoodieAvroFileReader {
@Override
public ClosableIterator<String> getRecordKeyIterator() {
- final Iterator<String> iterator = orcUtils.readRowKeys(conf, path).iterator();
+ final Iterator<String> iterator = orcUtils.readRowKeys(storage, path).iterator();
return new ClosableIterator<String>() {
@Override
public boolean hasNext() {
@@ -114,7 +114,7 @@ public class HoodieAvroOrcReader extends HoodieAvroFileReader {
@Override
public Schema getSchema() {
- return orcUtils.readAvroSchema(conf, path);
+ return orcUtils.readAvroSchema(storage, path);
}
@Override
@@ -123,6 +123,6 @@ public class HoodieAvroOrcReader extends HoodieAvroFileReader {
@Override
public long getTotalRecords() {
- return orcUtils.getRowCount(conf, path);
+ return orcUtils.getRowCount(storage, path);
}
}
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
index 6736379b4df..c1f7f7cb4b4 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
@@ -32,6 +32,7 @@ import org.apache.hudi.common.util.collection.CloseableMappingIterator;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieFileReader;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
@@ -58,14 +59,14 @@ import static org.apache.hudi.common.util.TypeUtils.unsafeCast;
public class HoodieAvroParquetReader extends HoodieAvroFileReader {
private final StoragePath path;
- private final StorageConfiguration<?> conf;
+ private final HoodieStorage storage;
private final FileFormatUtils parquetUtils;
private final List<ParquetReaderIterator> readerIterators = new ArrayList<>();
- public HoodieAvroParquetReader(StorageConfiguration<?> storageConf, StoragePath path) {
+ public HoodieAvroParquetReader(HoodieStorage storage, StoragePath path) {
// We have to clone the Hadoop Config as it might be subsequently modified
// by the Reader (for proper config propagation to Parquet components)
- this.conf = tryOverrideDefaultConfigs(storageConf.newInstance());
+ this.storage = storage.newInstance(path, tryOverrideDefaultConfigs(storage.getConf().newInstance()));
this.path = path;
this.parquetUtils = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET);
}
@@ -81,17 +82,17 @@ public class HoodieAvroParquetReader extends HoodieAvroFileReader {
@Override
public String[] readMinMaxRecordKeys() {
- return parquetUtils.readMinMaxRecordKeys(conf, path);
+ return parquetUtils.readMinMaxRecordKeys(storage, path);
}
@Override
public BloomFilter readBloomFilter() {
- return parquetUtils.readBloomFilterFromMetadata(conf, path);
+ return parquetUtils.readBloomFilterFromMetadata(storage, path);
}
@Override
public Set<Pair<String, Long>> filterRowKeys(Set<String> candidateRowKeys) {
- return parquetUtils.filterRowKeys(conf, path, candidateRowKeys);
+ return parquetUtils.filterRowKeys(storage, path, candidateRowKeys);
}
@Override
@@ -106,7 +107,7 @@ public class HoodieAvroParquetReader extends HoodieAvroFileReader {
@Override
public Schema getSchema() {
- return parquetUtils.readAvroSchema(conf, path);
+ return parquetUtils.readAvroSchema(storage, path);
}
@Override
@@ -116,7 +117,7 @@ public class HoodieAvroParquetReader extends HoodieAvroFileReader {
@Override
public long getTotalRecords() {
- return parquetUtils.getRowCount(conf, path);
+ return parquetUtils.getRowCount(storage, path);
}
private static StorageConfiguration<?> tryOverrideDefaultConfigs(StorageConfiguration<?> conf) {
@@ -163,7 +164,7 @@ public class HoodieAvroParquetReader extends HoodieAvroFileReader {
// NOTE: We have to set both Avro read-schema and projection schema to make
// sure that in case the file-schema is not equal to read-schema we'd still
// be able to read that file (in case projection is a proper one)
- Configuration hadoopConf = conf.unwrapAs(Configuration.class);
+ Configuration hadoopConf = storage.getConf().unwrapAs(Configuration.class);
if (!requestedSchema.isPresent()) {
AvroReadSupport.setAvroReadSchema(hadoopConf, schema);
AvroReadSupport.setRequestedProjection(hadoopConf, schema);
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieHadoopIOFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHadoopIOFactory.java
similarity index 81%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieHadoopIOFactory.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHadoopIOFactory.java
index c357a70be3e..4203fe90b4b 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieHadoopIOFactory.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHadoopIOFactory.java
@@ -17,16 +17,16 @@
* under the License.
*/
-package org.apache.hudi.io.storage;
+package org.apache.hudi.io.hadoop;
import org.apache.hudi.common.fs.ConsistencyGuard;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.io.hadoop.HoodieAvroFileReaderFactory;
-import org.apache.hudi.io.hadoop.HoodieAvroFileWriterFactory;
+import org.apache.hudi.io.storage.HoodieFileReaderFactory;
+import org.apache.hudi.io.storage.HoodieFileWriterFactory;
+import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
@@ -37,21 +37,21 @@ import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
*/
public class HoodieHadoopIOFactory extends HoodieIOFactory {
- public HoodieHadoopIOFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieHadoopIOFactory(HoodieStorage storage) {
+ super(storage);
}
@Override
public HoodieFileReaderFactory getReaderFactory(HoodieRecord.HoodieRecordType recordType) {
switch (recordType) {
case AVRO:
- return new HoodieAvroFileReaderFactory(storageConf);
+ return new HoodieAvroFileReaderFactory(storage);
case SPARK:
//TODO: remove this case [HUDI-7746]
try {
return (HoodieFileReaderFactory) ReflectionUtils
.loadClass("org.apache.hudi.io.storage.HoodieSparkFileReaderFactory",
- new Class<?>[] {StorageConfiguration.class}, storageConf);
+ new Class<?>[] {HoodieStorage.class}, storage);
} catch (Exception e) {
throw new HoodieException("Unable to create HoodieSparkFileReaderFactory", e);
}
@@ -64,13 +64,13 @@ public class HoodieHadoopIOFactory extends HoodieIOFactory {
public HoodieFileWriterFactory getWriterFactory(HoodieRecord.HoodieRecordType recordType) {
switch (recordType) {
case AVRO:
- return new HoodieAvroFileWriterFactory(storageConf);
+ return new HoodieAvroFileWriterFactory(storage);
case SPARK:
//TODO: remove this case [HUDI-7746]
try {
return (HoodieFileWriterFactory) ReflectionUtils
.loadClass("org.apache.hudi.io.storage.HoodieSparkFileWriterFactory",
- new Class<?>[] {StorageConfiguration.class}, storageConf);
+ new Class<?>[] {HoodieStorage.class}, storage);
} catch (Exception e) {
throw new HoodieException("Unable to create HoodieSparkFileWriterFactory", e);
}
@@ -81,7 +81,7 @@ public class HoodieHadoopIOFactory extends HoodieIOFactory {
@Override
public HoodieStorage getStorage(StoragePath storagePath) {
- return new HoodieHadoopStorage(storagePath, storageConf);
+ return storage.newInstance(storagePath, storage.getConf());
}
@Override
@@ -92,7 +92,7 @@ public class HoodieHadoopIOFactory extends HoodieIOFactory {
long initialRetryIntervalMs,
String retryExceptions,
ConsistencyGuard consistencyGuard) {
- return new HoodieHadoopStorage(path, storageConf, enableRetry, maxRetryIntervalMs,
+ return new HoodieHadoopStorage(path, storage.getConf(), enableRetry, maxRetryIntervalMs,
maxRetryNumbers, maxRetryIntervalMs, retryExceptions, consistencyGuard);
}
}
\ No newline at end of file
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
index f7df5e1f164..10826c7aace 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
@@ -59,7 +59,23 @@ public class HoodieHadoopStorage extends HoodieStorage {
private final FileSystem fs;
public HoodieHadoopStorage(StoragePath path, StorageConfiguration<?> conf) {
- this(HadoopFSUtils.getFs(path, conf.unwrapAs(Configuration.class)));
+ super(conf);
+ this.fs = HadoopFSUtils.getFs(path, conf.unwrapAs(Configuration.class));
+ }
+
+ public HoodieHadoopStorage(Path path, Configuration conf) {
+ super(HadoopFSUtils.getStorageConf(conf));
+ this.fs = HadoopFSUtils.getFs(path, conf);
+ }
+
+ public HoodieHadoopStorage(String path, Configuration conf) {
+ super(HadoopFSUtils.getStorageConf(conf));
+ this.fs = HadoopFSUtils.getFs(path, conf);
+ }
+
+ public HoodieHadoopStorage(String path, StorageConfiguration<?> conf) {
+ super(conf);
+ this.fs = HadoopFSUtils.getFs(path, conf);
}
public HoodieHadoopStorage(StoragePath path,
@@ -70,6 +86,7 @@ public class HoodieHadoopStorage extends HoodieStorage {
long initialRetryIntervalMs,
String retryExceptions,
ConsistencyGuard consistencyGuard) {
+ super(conf);
FileSystem fileSystem = getFs(path, conf.unwrapCopyAs(Configuration.class));
if (enableRetry) {
@@ -82,9 +99,15 @@ public class HoodieHadoopStorage extends HoodieStorage {
}
public HoodieHadoopStorage(FileSystem fs) {
+ super(new HadoopStorageConfiguration(fs.getConf()));
this.fs = fs;
}
+ @Override
+ public HoodieStorage newInstance(StoragePath path, StorageConfiguration<?> storageConf) {
+ return new HoodieHadoopStorage(path, storageConf);
+ }
+
@Override
public String getScheme() {
return fs.getScheme();
@@ -225,11 +248,6 @@ public class HoodieHadoopStorage extends HoodieStorage {
return fs;
}
- @Override
- public StorageConfiguration<Configuration> getConf() {
- return new HadoopStorageConfiguration(fs.getConf());
- }
-
@Override
public HoodieStorage getRawStorage() {
if (fs instanceof HoodieWrapperFileSystem) {
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
index 6db88b76005..63670bc21f9 100755
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
@@ -455,9 +455,9 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString());
- byte[] dataBlockContentBytes = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records, header).getContentBytes(storage.getConf());
+ byte[] dataBlockContentBytes = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records, header).getContentBytes(storage);
HoodieLogBlock.HoodieLogBlockContentLocation logBlockContentLoc = new HoodieLogBlock.HoodieLogBlockContentLocation(
- HoodieTestUtils.getDefaultStorageConfWithDefaults(), null, 0, dataBlockContentBytes.length, 0);
+ HoodieTestUtils.getStorage(basePath), null, 0, dataBlockContentBytes.length, 0);
HoodieDataBlock reusableDataBlock = new HoodieAvroDataBlock(null, Option.ofNullable(dataBlockContentBytes), false,
logBlockContentLoc, Option.ofNullable(getSimpleSchema()), header, new HashMap<>(), HoodieRecord.RECORD_KEY_METADATA_FIELD);
long writtenSize = 0;
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
index 69c4d35a847..401dde54997 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
@@ -29,8 +29,8 @@ import org.apache.hudi.common.testutils.SchemaTestUtil;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.internal.schema.HoodieSchemaException;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
@@ -95,7 +95,7 @@ public class TestTableSchemaResolver {
StoragePath partitionPath = new StoragePath(testDir, "partition1");
Schema expectedSchema = getSimpleSchema();
StoragePath logFilePath = writeLogFile(partitionPath, expectedSchema);
- assertEquals(expectedSchema, TableSchemaResolver.readSchemaFromLogFile(HoodieStorageUtils.getStorage(
+ assertEquals(expectedSchema, TableSchemaResolver.readSchemaFromLogFile(new HoodieHadoopStorage(
logFilePath, HoodieTestUtils.getDefaultStorageConfWithDefaults()), logFilePath));
}
@@ -106,8 +106,7 @@ public class TestTableSchemaResolver {
}
private StoragePath writeLogFile(StoragePath partitionPath, Schema schema) throws IOException, URISyntaxException, InterruptedException {
- HoodieStorage storage = HoodieStorageUtils.getStorage(
- partitionPath, HoodieTestUtils.getDefaultStorageConfWithDefaults());
+ HoodieStorage storage = HoodieTestUtils.getStorage(partitionPath);
HoodieLogFormat.Writer writer =
HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath).withFileExtension(HoodieLogFile.DELTA_EXTENSION)
.withFileId("test-fileid1").withDeltaCommit("100").withStorage(storage).build();
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieDeleteBlock.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieDeleteBlock.java
index a6bc014b3f4..b32b865a19a 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieDeleteBlock.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieDeleteBlock.java
@@ -125,7 +125,7 @@ public class TestHoodieDeleteBlock {
deleteRecordList.add(Pair.of(dr, -1L));
}
HoodieDeleteBlock deleteBlock = new HoodieDeleteBlock(deleteRecordList, false, new HashMap<>());
- byte[] contentBytes = deleteBlock.getContentBytes(HoodieTestUtils.getDefaultStorageConf());
+ byte[] contentBytes = deleteBlock.getContentBytes(HoodieTestUtils.getDefaultStorage());
HoodieDeleteBlock deserializeDeleteBlock = new HoodieDeleteBlock(
Option.of(contentBytes), null, true, Option.empty(), new HashMap<>(), new HashMap<>());
DeleteRecord[] deserializedDeleteRecords = deserializeDeleteBlock.getRecordsToDelete();
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestHarness.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestHarness.java
index e4abe4ce9c5..5d50f885204 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestHarness.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestHarness.java
@@ -30,6 +30,7 @@ import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.FileSystem;
@@ -117,7 +118,7 @@ public class HoodieFileGroupReaderTestHarness extends HoodieCommonTestHarness {
0L,
Long.MAX_VALUE,
properties,
- storageConf,
+ new HoodieHadoopStorage(basePath, storageConf),
tableConfig,
readerContext
);
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
index f6caa31a62c..e4f316170a9 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
@@ -24,11 +24,8 @@ import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.minicluster.HdfsTestService;
import org.apache.hudi.exception.HoodieIOException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -188,8 +185,7 @@ public class TestDFSPropertiesConfiguration {
ENVIRONMENT_VARIABLES.clear(DFSPropertiesConfiguration.CONF_FILE_DIR_ENV_NAME);
DFSPropertiesConfiguration.refreshGlobalProps();
try {
- if (!HoodieStorageUtils.getStorage(
- DFSPropertiesConfiguration.DEFAULT_PATH, HadoopFSUtils.getStorageConf(new Configuration()))
+ if (!HoodieTestUtils.getStorage(DFSPropertiesConfiguration.DEFAULT_PATH)
.exists(DFSPropertiesConfiguration.DEFAULT_PATH)) {
assertEquals(0, DFSPropertiesConfiguration.getGlobalProps().size());
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
index e39bd25f46a..248350c7a82 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
@@ -98,13 +98,13 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
// Read and verify
List<String> rowKeysInFile = new ArrayList<>(parquetUtils.readRowKeys(
- HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath)));
+ HoodieTestUtils.getStorage(filePath), new StoragePath(filePath)));
Collections.sort(rowKeysInFile);
Collections.sort(rowKeys);
assertEquals(rowKeys, rowKeysInFile, "Did not read back the expected list of keys");
BloomFilter filterInFile = parquetUtils.readBloomFilterFromMetadata(
- HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath));
+ HoodieTestUtils.getStorage(filePath), new StoragePath(filePath));
for (String rowKey : rowKeys) {
assertTrue(filterInFile.mightContain(rowKey), "key should be found in bloom filter");
}
@@ -128,7 +128,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
// Read and verify
Set<Pair<String, Long>> filtered = parquetUtils.filterRowKeys(
- HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath), filter);
+ HoodieTestUtils.getStorage(filePath), new StoragePath(filePath), filter);
assertEquals(filter.size(), filtered.size(), "Filtered count does not match");
@@ -155,7 +155,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
// Read and verify
List<Pair<HoodieKey, Long>> fetchedRows = parquetUtils.fetchRecordKeysWithPositions(
- HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath));
+ HoodieTestUtils.getStorage(filePath), new StoragePath(filePath));
assertEquals(rowKeys.size(), fetchedRows.size(), "Total count does not match");
for (Pair<HoodieKey, Long> entry : fetchedRows) {
@@ -181,7 +181,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
// Read and verify
List<Pair<HoodieKey, Long>> fetchedRows = parquetUtils.fetchRecordKeysWithPositions(
- HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath),
+ HoodieTestUtils.getStorage(filePath), new StoragePath(filePath),
Option.of(new TestBaseKeyGen("abc", "def")));
assertEquals(rowKeys.size(), fetchedRows.size(), "Total count does not match");
@@ -200,7 +200,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
writeParquetFile(BloomFilterTypeCode.SIMPLE.name(), filePath, rowKeys);
assertEquals(123, parquetUtils.getRowCount(
- HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath)));
+ HoodieTestUtils.getStorage(filePath), new StoragePath(filePath)));
}
@Test
@@ -266,7 +266,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
columnList.add(dataField);
List<HoodieColumnRangeMetadata<Comparable>> columnRangeMetadataList = parquetUtils.readColumnStatsFromMetadata(
- HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath), columnList)
+ HoodieTestUtils.getStorage(filePath), new StoragePath(filePath), columnList)
.stream()
.sorted(Comparator.comparing(HoodieColumnRangeMetadata::getColumnName))
.collect(Collectors.toList());
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java
index b46e0747aeb..1bb2324505a 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java
@@ -20,14 +20,13 @@
package org.apache.hudi.io.hadoop;
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.io.storage.HoodieIOFactory;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
-import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -47,9 +46,9 @@ public class TestHoodieAvroFileReaderFactory {
@Test
public void testGetFileReader() throws IOException {
// parquet file format.
- final StorageConfiguration<?> storageConf = HadoopFSUtils.getStorageConf(new Configuration());
+ final HoodieStorage storage = HoodieTestUtils.getDefaultStorage();
final StoragePath parquetPath = new StoragePath("/partition/path/f1_1-0-1_000.parquet");
- HoodieFileReader parquetReader = HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecordType.AVRO)
+ HoodieFileReader parquetReader = HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, parquetPath);
assertTrue(parquetReader instanceof HoodieAvroParquetReader);
@@ -57,14 +56,14 @@ public class TestHoodieAvroFileReaderFactory {
final StoragePath logPath = new StoragePath(
"/partition/path/f.b51192a8-574b-4a85-b246-bcfec03ac8bf_100.log.2_1-0-1");
final Throwable thrown = assertThrows(UnsupportedOperationException.class, () -> {
- HoodieFileReader logWriter = HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecordType.AVRO)
+ HoodieFileReader logWriter = HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, logPath);
}, "should fail since log storage reader is not supported yet.");
assertTrue(thrown.getMessage().contains("format not supported yet."));
// Orc file format.
final StoragePath orcPath = new StoragePath("/partition/path/f1_1-0-1_000.orc");
- HoodieFileReader orcReader = HoodieIOFactory.getIOFactory(storageConf)
+ HoodieFileReader orcReader = HoodieIOFactory.getIOFactory(storage)
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, orcPath);
assertTrue(orcReader instanceof HoodieAvroOrcReader);
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHBaseHFileReaderWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHBaseHFileReaderWriter.java
index 8db8ffa4a39..e3743b5255a 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHBaseHFileReaderWriter.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHBaseHFileReaderWriter.java
@@ -19,13 +19,12 @@
package org.apache.hudi.io.hadoop;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -67,16 +66,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
public class TestHoodieHBaseHFileReaderWriter extends TestHoodieHFileReaderWriterBase {
@Override
protected HoodieAvroFileReader createReader(
- StorageConfiguration<?> conf) throws Exception {
- return new HoodieHBaseAvroHFileReader(conf, getFilePath(), Option.empty());
+ HoodieStorage storage) throws Exception {
+ return new HoodieHBaseAvroHFileReader(storage.getConf(), getFilePath(), Option.empty());
}
@Override
- protected HoodieAvroHFileReaderImplBase createHFileReader(StorageConfiguration<?> conf,
+ protected HoodieAvroHFileReaderImplBase createHFileReader(HoodieStorage storage,
byte[] content) throws IOException {
FileSystem fs = HadoopFSUtils.getFs(getFilePath().toString(), new Configuration());
- return new HoodieHBaseAvroHFileReader(conf, new StoragePath(DUMMY_BASE_PATH),
- HoodieStorageUtils.getStorage(getFilePath(), conf), content, Option.empty());
+ return new HoodieHBaseAvroHFileReader(storage.getConf(), new StoragePath(DUMMY_BASE_PATH), storage, content, Option.empty());
}
@Override
@@ -85,8 +83,7 @@ public class TestHoodieHBaseHFileReaderWriter extends TestHoodieHFileReaderWrite
boolean mayUseDefaultComparator,
Class<?> expectedComparatorClazz,
int count) throws IOException {
- HoodieStorage storage = HoodieStorageUtils.getStorage(
- getFilePath(), HadoopFSUtils.getStorageConf(new Configuration()));
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
try (HFile.Reader reader =
HoodieHFileUtils.createHFileReader(storage, new StoragePath(DUMMY_BASE_PATH), content)) {
// HFile version is 3
@@ -106,7 +103,7 @@ public class TestHoodieHBaseHFileReaderWriter extends TestHoodieHFileReaderWrite
public void testReaderGetRecordIteratorByKeysWithBackwardSeek() throws Exception {
writeFileWithSimpleSchema();
try (HoodieAvroHFileReaderImplBase hfileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
List<GenericRecord> allRecords = toStream(hfileReader.getRecordIterator())
@@ -184,7 +181,7 @@ public class TestHoodieHBaseHFileReaderWriter extends TestHoodieHFileReaderWrite
writer.close();
try (HoodieAvroHFileReaderImplBase hFileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hFileReader);
assertEquals(recordMap.values().stream().flatMap(List::stream).collect(Collectors.toList()), records);
}
@@ -199,7 +196,7 @@ public class TestHoodieHBaseHFileReaderWriter extends TestHoodieHFileReaderWrite
List<GenericRecord> expectedRecords = rowsList.stream().flatMap(row -> recordMap.get(row).stream()).collect(Collectors.toList());
try (HoodieAvroHFileReaderImplBase hFileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
List<GenericRecord> result = HoodieAvroHFileReaderImplBase.readRecords(hFileReader, rowsList).stream().map(r -> (GenericRecord) r).collect(Collectors.toList());
assertEquals(expectedRecords, result);
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
index b87af2c8371..362f58a00cf 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
@@ -19,16 +19,15 @@
package org.apache.hudi.io.hadoop;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieNativeAvroHFileReader;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.Test;
import java.io.IOException;
@@ -47,14 +46,14 @@ public class TestHoodieHFileReaderWriter extends TestHoodieHFileReaderWriterBase
@Override
protected HoodieAvroFileReader createReader(
- StorageConfiguration<?> conf) throws Exception {
- return new HoodieNativeAvroHFileReader(conf, getFilePath(), Option.empty());
+ HoodieStorage storage) throws Exception {
+ return new HoodieNativeAvroHFileReader(storage, getFilePath(), Option.empty());
}
@Override
- protected HoodieAvroHFileReaderImplBase createHFileReader(StorageConfiguration<?> conf,
+ protected HoodieAvroHFileReaderImplBase createHFileReader(HoodieStorage storage,
byte[] content) throws IOException {
- return new HoodieNativeAvroHFileReader(conf, content, Option.empty());
+ return new HoodieNativeAvroHFileReader(storage, content, Option.empty());
}
@Override
@@ -63,8 +62,7 @@ public class TestHoodieHFileReaderWriter extends TestHoodieHFileReaderWriterBase
boolean mayUseDefaultComparator,
Class<?> expectedComparatorClazz,
int count) throws IOException {
- try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(
- HadoopFSUtils.getStorageConf(new Configuration()), content)) {
+ try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(HoodieTestUtils.getStorage(hfileName), content)) {
assertEquals(count, hfileReader.getTotalRecords());
}
}
@@ -73,7 +71,7 @@ public class TestHoodieHFileReaderWriter extends TestHoodieHFileReaderWriterBase
public void testReaderGetRecordIteratorByKeysWithBackwardSeek() throws Exception {
writeFileWithSimpleSchema();
try (HoodieAvroHFileReaderImplBase hfileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
// Filter for "key00001, key05, key24, key16, key31, key61".
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriterBase.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriterBase.java
index 3bdd561a282..caf09bb94e4 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriterBase.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriterBase.java
@@ -27,14 +27,13 @@ import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -74,11 +73,11 @@ import java.util.stream.StreamSupport;
import static org.apache.hudi.common.testutils.FileSystemTestUtils.RANDOM;
import static org.apache.hudi.common.testutils.SchemaTestUtil.getSchemaFromResource;
import static org.apache.hudi.common.util.CollectionUtils.toStream;
+import static org.apache.hudi.io.hadoop.HoodieHFileConfig.HFILE_COMPARATOR;
import static org.apache.hudi.io.hfile.TestHFileReader.BOOTSTRAP_INDEX_HFILE_SUFFIX;
import static org.apache.hudi.io.hfile.TestHFileReader.COMPLEX_SCHEMA_HFILE_SUFFIX;
import static org.apache.hudi.io.hfile.TestHFileReader.SIMPLE_SCHEMA_HFILE_SUFFIX;
import static org.apache.hudi.io.hfile.TestHFileReader.readHFileFromResources;
-import static org.apache.hudi.io.hadoop.HoodieHFileConfig.HFILE_COMPARATOR;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -92,7 +91,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
// Number of records in HFile fixtures for compatibility tests
protected static final int NUM_RECORDS_FIXTURE = 50;
- protected abstract HoodieAvroHFileReaderImplBase createHFileReader(StorageConfiguration<?> conf,
+ protected abstract HoodieAvroHFileReaderImplBase createHFileReader(HoodieStorage storage,
byte[] content) throws IOException;
protected abstract void verifyHFileReader(byte[] content,
@@ -114,7 +113,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
protected HoodieAvroHFileWriter createWriter(
Schema avroSchema, boolean populateMetaFields) throws Exception {
String instantTime = "000";
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
Properties props = new Properties();
props.setProperty(HoodieTableConfig.POPULATE_META_FIELDS.key(), Boolean.toString(populateMetaFields));
TaskContextSupplier mockTaskContextSupplier = Mockito.mock(TaskContextSupplier.class);
@@ -123,7 +122,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
when(partitionSupplier.get()).thenReturn(10);
return (HoodieAvroHFileWriter) HoodieFileWriterFactory.getFileWriter(
- instantTime, getFilePath(), conf, HoodieStorageConfig.newBuilder().fromProperties(props).build(), avroSchema,
+ instantTime, getFilePath(), storage, HoodieStorageConfig.newBuilder().fromProperties(props).build(), avroSchema,
mockTaskContextSupplier, HoodieRecord.HoodieRecordType.AVRO);
}
@@ -133,8 +132,8 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
}
@Override
- protected void verifyMetadata(StorageConfiguration<?> conf) throws IOException {
- try (HoodieAvroFileReader reader = createReader(conf)) {
+ protected void verifyMetadata(HoodieStorage storage) throws IOException {
+ try (HoodieAvroFileReader reader = createReader(storage)) {
assertEquals(NUM_RECORDS, reader.getTotalRecords());
} catch (Exception e) {
throw new RuntimeException(e);
@@ -142,8 +141,8 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
}
@Override
- protected void verifySchema(StorageConfiguration<?> conf, String schemaPath) throws IOException {
- try (HoodieAvroFileReader reader = createReader(conf)) {
+ protected void verifySchema(HoodieStorage storage, String schemaPath) throws IOException {
+ try (HoodieAvroFileReader reader = createReader(storage)) {
assertEquals(
getSchemaFromResource(TestHoodieHBaseHFileReaderWriter.class, schemaPath),
reader.getSchema());
@@ -180,9 +179,9 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
}
writer.close();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
HoodieAvroHFileReaderImplBase hoodieHFileReader =
- (HoodieAvroHFileReaderImplBase) createReader(conf);
+ (HoodieAvroHFileReaderImplBase) createReader(storage);
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
assertEquals(new ArrayList<>(recordMap.values()), records);
@@ -198,7 +197,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
List<GenericRecord> expectedRecords =
rowsList.stream().map(recordMap::get).collect(Collectors.toList());
- hoodieHFileReader = (HoodieAvroHFileReaderImplBase) createReader(conf);
+ hoodieHFileReader = (HoodieAvroHFileReaderImplBase) createReader(storage);
List<GenericRecord> result =
HoodieAvroHFileReaderImplBase.readRecords(hoodieHFileReader, rowsList).stream()
.map(r -> (GenericRecord) r).collect(Collectors.toList());
@@ -226,12 +225,11 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
@Test
public void testReadHFileFormatRecords() throws Exception {
writeFileWithSimpleSchema();
- HoodieStorage storage = HoodieStorageUtils.getStorage(getFilePath(), HadoopFSUtils.getStorageConf(new Configuration()));
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
byte[] content = FileIOUtils.readAsByteArray(
storage.open(getFilePath()), (int) storage.getPathInfo(getFilePath()).getLength());
// Reading byte array in HFile format, without actual file path
- StorageConfiguration<?> storageConf = storage.getConf();
- try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storageConf, content)) {
+ try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storage, content)) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
assertEquals(NUM_RECORDS, hfileReader.getTotalRecords());
@@ -243,7 +241,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
public void testReaderGetRecordIterator() throws Exception {
writeFileWithSimpleSchema();
try (HoodieAvroHFileReaderImplBase hfileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
List<String> keys =
IntStream.concat(IntStream.range(40, NUM_RECORDS * 2), IntStream.range(10, 20))
.mapToObj(i -> "key" + String.format("%02d", i)).collect(Collectors.toList());
@@ -271,7 +269,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
public void testReaderGetRecordIteratorByKeys() throws Exception {
writeFileWithSimpleSchema();
try (HoodieAvroHFileReaderImplBase hfileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
@@ -315,7 +313,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
public void testReaderGetRecordIteratorByKeyPrefixes() throws Exception {
writeFileWithSimpleSchema();
try (HoodieAvroHFileReaderImplBase hfileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
@@ -455,8 +453,8 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
verifyHFileReader(
content, hfilePrefix, true, HFILE_COMPARATOR.getClass(), NUM_RECORDS_FIXTURE);
- StorageConfiguration storageConf = HadoopFSUtils.getStorageConf(fs.getConf());
- try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storageConf, content)) {
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storage, content)) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
assertEquals(NUM_RECORDS_FIXTURE, hfileReader.getTotalRecords());
@@ -466,7 +464,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
content = readHFileFromResources(complexHFile);
verifyHFileReader(
content, hfilePrefix, true, HFILE_COMPARATOR.getClass(), NUM_RECORDS_FIXTURE);
- try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storageConf, content)) {
+ try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storage, content)) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchemaWithUDT.avsc");
assertEquals(NUM_RECORDS_FIXTURE, hfileReader.getTotalRecords());
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java
index 314334365b2..0b735ce9f16 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java
@@ -30,6 +30,7 @@ import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.io.storage.HoodieOrcConfig;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
@@ -77,15 +78,15 @@ public class TestHoodieOrcReaderWriter extends TestHoodieReaderWriterBase {
@Override
protected HoodieAvroFileReader createReader(
- StorageConfiguration<?> conf) throws Exception {
- return (HoodieAvroFileReader) HoodieIOFactory.getIOFactory(conf).getReaderFactory(HoodieRecordType.AVRO)
+ HoodieStorage storage) throws Exception {
+ return (HoodieAvroFileReader) HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, getFilePath());
}
@Override
- protected void verifyMetadata(StorageConfiguration<?> conf) throws IOException {
+ protected void verifyMetadata(HoodieStorage storage) throws IOException {
Reader orcReader = OrcFile.createReader(
- new Path(getFilePath().toUri()), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)));
+ new Path(getFilePath().toUri()), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)));
assertEquals(4, orcReader.getMetadataKeys().size());
assertTrue(orcReader.getMetadataKeys().contains(HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER));
assertTrue(orcReader.getMetadataKeys().contains(HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER));
@@ -96,9 +97,9 @@ public class TestHoodieOrcReaderWriter extends TestHoodieReaderWriterBase {
}
@Override
- protected void verifySchema(StorageConfiguration<?> conf, String schemaPath) throws IOException {
+ protected void verifySchema(HoodieStorage storage, String schemaPath) throws IOException {
Reader orcReader = OrcFile.createReader(
- new Path(getFilePath().toUri()), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)));
+ new Path(getFilePath().toUri()), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)));
if ("/exampleSchema.avsc".equals(schemaPath)) {
assertEquals("struct<_row_key:string,time:string,number:int>",
orcReader.getSchema().toString());
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieReaderWriterBase.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieReaderWriterBase.java
index 3fd0ad80319..2fa13af817a 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieReaderWriterBase.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieReaderWriterBase.java
@@ -24,21 +24,20 @@ import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.collection.Pair;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieAvroFileWriter;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileWriter;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -78,11 +77,11 @@ public abstract class TestHoodieReaderWriterBase {
Schema avroSchema, boolean populateMetaFields) throws Exception;
protected abstract HoodieAvroFileReader createReader(
- StorageConfiguration<?> conf) throws Exception;
+ HoodieStorage storage) throws Exception;
- protected abstract void verifyMetadata(StorageConfiguration<?> conf) throws IOException;
+ protected abstract void verifyMetadata(HoodieStorage storage) throws IOException;
- protected abstract void verifySchema(StorageConfiguration<?> conf, String schemaPath) throws IOException;
+ protected abstract void verifySchema(HoodieStorage storage, String schemaPath) throws IOException;
@BeforeEach
@AfterEach
@@ -98,10 +97,10 @@ public abstract class TestHoodieReaderWriterBase {
Schema avroSchema = getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
writeFileWithSimpleSchema();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
- verifyMetadata(conf);
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ verifyMetadata(storage);
- try (HoodieAvroFileReader hoodieReader = createReader(conf)) {
+ try (HoodieAvroFileReader hoodieReader = createReader(storage)) {
BloomFilter filter = hoodieReader.readBloomFilter();
for (int i = 0; i < NUM_RECORDS; i++) {
String key = "key" + String.format("%02d", i);
@@ -122,10 +121,10 @@ public abstract class TestHoodieReaderWriterBase {
String schemaPath = "/exampleSchema.avsc";
writeFileWithSimpleSchema();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
- verifyMetadata(conf);
- verifySchema(conf, schemaPath);
- verifySimpleRecords(createReader(conf).getRecordIterator());
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ verifyMetadata(storage);
+ verifySchema(storage, schemaPath);
+ verifySimpleRecords(createReader(storage).getRecordIterator());
}
@Test
@@ -149,10 +148,10 @@ public abstract class TestHoodieReaderWriterBase {
}
writer.close();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
- verifyMetadata(conf);
- verifySchema(conf, schemaPath);
- verifyComplexRecords(createReader(conf).getRecordIterator());
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ verifyMetadata(storage);
+ verifySchema(storage, schemaPath);
+ verifyComplexRecords(createReader(storage).getRecordIterator());
}
@ParameterizedTest
@@ -165,8 +164,8 @@ public abstract class TestHoodieReaderWriterBase {
})
public void testWriteReadWithEvolvedSchema(String evolvedSchemaPath) throws Exception {
writeFileWithSimpleSchema();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
- try (HoodieAvroFileReader hoodieReader = createReader(conf)) {
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ try (HoodieAvroFileReader hoodieReader = createReader(storage)) {
verifyReaderWithSchema(evolvedSchemaPath, hoodieReader);
}
}
@@ -174,9 +173,9 @@ public abstract class TestHoodieReaderWriterBase {
@Test
public void testReaderFilterRowKeys() throws Exception {
writeFileWithSchemaWithMeta();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
- verifyMetadata(conf);
- verifyFilterRowKeys(createReader(conf));
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ verifyMetadata(storage);
+ verifyFilterRowKeys(createReader(storage));
}
protected void writeFileWithSimpleSchema() throws Exception {
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
index 059d77158cc..42e88ff22a8 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
@@ -72,8 +72,9 @@ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness {
HoodieLocalEngineContext localEngineContext =
new HoodieLocalEngineContext(metaClient.getStorageConf());
FileSystemBackedTableMetadata fileSystemBackedTableMetadata =
- new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(),
- metaClient.getStorageConf(), basePath);
+ new FileSystemBackedTableMetadata(localEngineContext,
+ metaClient.getTableConfig(),
+ metaClient.getStorage(), basePath);
Assertions.assertEquals(0, fileSystemBackedTableMetadata.getAllPartitionPaths().size());
Assertions.assertEquals(10,
fileSystemBackedTableMetadata.getAllFilesInPartition(new StoragePath(basePath)).size());
@@ -103,7 +104,7 @@ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness {
new HoodieLocalEngineContext(metaClient.getStorageConf());
FileSystemBackedTableMetadata fileSystemBackedTableMetadata =
new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(),
- metaClient.getStorageConf(), basePath);
+ metaClient.getStorage(), basePath);
Assertions.assertEquals(3, fileSystemBackedTableMetadata.getAllPartitionPaths().size());
List<String> fullPartitionPaths =
@@ -132,7 +133,7 @@ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness {
new HoodieLocalEngineContext(metaClient.getStorageConf());
FileSystemBackedTableMetadata fileSystemBackedTableMetadata =
new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(),
- metaClient.getStorageConf(), basePath);
+ metaClient.getStorage(), basePath);
Assertions.assertEquals(3, fileSystemBackedTableMetadata.getAllPartitionPaths().size());
Assertions.assertEquals(10, fileSystemBackedTableMetadata.getAllFilesInPartition(
new StoragePath(basePath + "/" + ONE_LEVEL_PARTITIONS.get(0))).size());
@@ -163,7 +164,7 @@ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness {
new HoodieLocalEngineContext(metaClient.getStorageConf());
FileSystemBackedTableMetadata fileSystemBackedTableMetadata =
new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(),
- metaClient.getStorageConf(), basePath);
+ metaClient.getStorage(), basePath);
Assertions.assertEquals(3, fileSystemBackedTableMetadata.getAllPartitionPaths().size());
Assertions.assertEquals(10, fileSystemBackedTableMetadata.getAllFilesInPartition(
new StoragePath(basePath + "/" + MULTI_LEVEL_PARTITIONS.get(0))).size());
@@ -193,7 +194,7 @@ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness {
new HoodieLocalEngineContext(metaClient.getStorageConf());
FileSystemBackedTableMetadata fileSystemBackedTableMetadata =
new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(),
- metaClient.getStorageConf(), basePath);
+ metaClient.getStorage(), basePath);
Assertions.assertEquals(3, fileSystemBackedTableMetadata.getAllPartitionPaths().size());
Assertions.assertEquals(0, fileSystemBackedTableMetadata.getAllFilesInPartition(
new StoragePath(basePath + "/" + MULTI_LEVEL_PARTITIONS.get(0))).size());
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
index b69dc94609b..2dc4c39ecc3 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
@@ -216,7 +216,7 @@ public class TestHoodieTableMetadataUtil extends HoodieCommonTestHarness {
HoodieFileWriter writer = HoodieFileWriterFactory.getFileWriter(
instant,
path,
- metaClient.getStorageConf(),
+ metaClient.getStorage(),
metaClient.getTableConfig(),
HoodieTestDataGenerator.AVRO_SCHEMA_WITH_METADATA_FIELDS,
engineContext.getTaskContextSupplier(),
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileRecordReader.java
index 1d057901908..3414f5fdea8 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileRecordReader.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileRecordReader.java
@@ -29,6 +29,7 @@ import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
@@ -59,7 +60,7 @@ public class HoodieHFileRecordReader implements RecordReader<NullWritable, Array
StoragePath path = convertToStoragePath(fileSplit.getPath());
StorageConfiguration<?> storageConf = HadoopFSUtils.getStorageConf(conf);
HoodieConfig hoodieConfig = getReaderConfigs(storageConf);
- reader = HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
+ reader = HoodieIOFactory.getIOFactory(new HoodieHadoopStorage(path, storageConf)).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, path, HoodieFileFormat.HFILE, Option.empty());
schema = reader.getSchema();
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
index 51d8a9f3af4..3b465abd724 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
@@ -30,9 +30,9 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.hadoop.utils.HoodieHiveUtils;
import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -133,8 +133,7 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
Path folder = null;
try {
if (storage == null) {
- storage =
- HoodieStorageUtils.getStorage(convertToStoragePath(path), conf);
+ storage = new HoodieHadoopStorage(convertToStoragePath(path), conf);
}
// Assumes path is a file
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java
index 6945b241e0a..fe88855d458 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java
@@ -48,6 +48,7 @@ import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -524,10 +525,10 @@ public class HoodieInputFormatUtils {
List<HoodieCommitMetadata> metadataList) {
// TODO: Use HoodieMetaTable to extract affected file directly.
HashMap<String, StoragePathInfo> fullPathToInfoMap = new HashMap<>();
+ HoodieStorage storage = new HoodieHadoopStorage(basePath, HadoopFSUtils.getStorageConf(hadoopConf));
// Iterate through the given commits.
for (HoodieCommitMetadata metadata : metadataList) {
- fullPathToInfoMap.putAll(metadata.getFullPathToInfo(
- HadoopFSUtils.getStorageConf(hadoopConf), basePath.toString()));
+ fullPathToInfoMap.putAll(metadata.getFullPathToInfo(storage, basePath.toString()));
}
return new ArrayList<>(fullPathToInfoMap.values());
}
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
index 9cd25580d10..07863ac5b97 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
@@ -28,6 +28,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.avro.JsonProperties;
import org.apache.avro.LogicalType;
@@ -312,7 +313,8 @@ public class HoodieRealtimeRecordReaderUtils {
public static HoodieFileReader getBaseFileReader(Path path, JobConf conf) throws IOException {
StorageConfiguration<?> storageConf = HadoopFSUtils.getStorageConf(conf);
HoodieConfig hoodieConfig = getReaderConfigs(storageConf);
- return HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
+ return HoodieIOFactory.getIOFactory(new HoodieHadoopStorage(path, conf))
+ .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, convertToStoragePath(path));
}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
index 9de894fb652..b8e02f06a41 100644
--- a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
@@ -97,7 +97,7 @@ public class DFSHoodieDatasetInputReader extends DFSDeltaInputReader {
// Using FSUtils.getFS here instead of metaClient.getFS() since we don't want to count these listStatus
// calls in metrics as they are not part of normal HUDI operation.
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
- List<String> partitionPaths = FSUtils.getAllPartitionPaths(engineContext, metaClient.getBasePath(),
+ List<String> partitionPaths = FSUtils.getAllPartitionPaths(engineContext, metaClient.getStorage(), metaClient.getBasePath(),
HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS);
// Sort partition so we can pick last N partitions by default
Collections.sort(partitionPaths);
@@ -274,7 +274,7 @@ public class DFSHoodieDatasetInputReader extends DFSDeltaInputReader {
if (fileSlice.getBaseFile().isPresent()) {
// Read the base files using the latest writer schema.
Schema schema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(schemaStr));
- HoodieAvroFileReader reader = TypeUtils.unsafeCast(HoodieIOFactory.getIOFactory(metaClient.getStorageConf())
+ HoodieAvroFileReader reader = TypeUtils.unsafeCast(HoodieIOFactory.getIOFactory(metaClient.getStorage())
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(
DEFAULT_HUDI_CONFIG_FOR_READER,
diff --git a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
index a92a2938bfc..fc5ae52de54 100644
--- a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
+++ b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
@@ -47,6 +47,21 @@ import java.util.UUID;
public abstract class HoodieStorage implements Closeable {
public static final Logger LOG = LoggerFactory.getLogger(HoodieStorage.class);
+ protected final StorageConfiguration<?> storageConf;
+
+ public HoodieStorage(StorageConfiguration<?> storageConf) {
+ this.storageConf = storageConf;
+ }
+
+ /**
+ * @param path path to instantiate the storage.
+ * @param storageConf new storage configuration.
+ * @return new {@link HoodieStorage} instance with the configuration.
+ */
+ @PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
+ public abstract HoodieStorage newInstance(StoragePath path,
+ StorageConfiguration<?> storageConf);
+
/**
* @return the scheme of the storage.
*/
@@ -257,16 +272,18 @@ public abstract class HoodieStorage implements Closeable {
public abstract Object getFileSystem();
/**
- * @return the storage configuration.
+ * @return the raw storage.
*/
@PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
- public abstract StorageConfiguration<?> getConf();
+ public abstract HoodieStorage getRawStorage();
/**
- * @return the raw storage.
+ * @return the storage configuration.
*/
@PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
- public abstract HoodieStorage getRawStorage();
+ public final StorageConfiguration<?> getConf() {
+ return storageConf;
+ }
/**
* Creates a new file with overwrite set to false. This ensures files are created
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java
index ad17abd294a..d237c2d6560 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java
@@ -48,7 +48,9 @@ public class DatasetBulkInsertOverwriteTableCommitActionExecutor extends Dataset
@Override
protected Map<String, List<String>> getPartitionToReplacedFileIds(HoodieData<WriteStatus> writeStatuses) {
HoodieEngineContext context = writeClient.getEngineContext();
- List<String> partitionPaths = FSUtils.getAllPartitionPaths(context, writeConfig.getMetadataConfig(),
+ List<String> partitionPaths = FSUtils.getAllPartitionPaths(context,
+ table.getStorage(),
+ writeConfig.getMetadataConfig(),
table.getMetaClient().getBasePathV2().toString());
if (partitionPaths == null || partitionPaths.isEmpty()) {
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
index 8f54ff300e2..facf09e4fe2 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
@@ -40,18 +40,20 @@ import org.apache.hudi.config.HoodieBootstrapConfig.DATA_QUERIES_ONLY
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.exception.HoodieException
import org.apache.hudi.hadoop.fs.HadoopFSUtils
+import org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
import org.apache.hudi.io.storage.HoodieSparkIOFactory
import org.apache.hudi.metadata.HoodieTableMetadata
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hudi.storage.{StoragePath, StoragePathInfo}
+
import org.apache.avro.Schema
import org.apache.avro.generic.GenericRecord
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileStatus, Path}
import org.apache.hadoop.mapred.JobConf
-import org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath
import org.apache.spark.execution.datasources.HoodieInMemoryFileIndex
import org.apache.spark.internal.Logging
import org.apache.spark.rdd.RDD
@@ -854,7 +856,8 @@ object HoodieBaseRelation extends SparkAdapterSupport {
val hoodieConfig = new HoodieConfig()
hoodieConfig.setValue(USE_NATIVE_HFILE_READER,
options.getOrElse(USE_NATIVE_HFILE_READER.key(), USE_NATIVE_HFILE_READER.defaultValue().toString))
- val reader = new HoodieSparkIOFactory(storageConf).getReaderFactory(HoodieRecordType.AVRO)
+ val reader = new HoodieSparkIOFactory(new HoodieHadoopStorage(filePath, storageConf))
+ .getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, filePath, HFILE)
val requiredRowSchema = requiredDataSchema.structTypeSchema
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
index 87418764dea..97a29f9b019 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
@@ -54,6 +54,7 @@ import org.apache.hudi.keygen.constant.KeyGeneratorType
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory
import org.apache.hudi.keygen.{BaseKeyGenerator, TimestampBasedAvroKeyGenerator, TimestampBasedKeyGenerator}
import org.apache.hudi.metrics.Metrics
+import org.apache.hudi.storage.HoodieStorage
import org.apache.hudi.sync.common.HoodieSyncConfig
import org.apache.hudi.sync.common.util.SyncUtilHelpers
import org.apache.hudi.sync.common.util.SyncUtilHelpers.getHoodieMetaSyncException
@@ -396,20 +397,20 @@ class HoodieSparkSqlWriterInternal {
}
val keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(new TypedProperties(hoodieConfig.getProps))
+ val tableMetaClient = HoodieTableMetaClient.builder
+ .setConf(HadoopFSUtils.getStorageConfWithCopy(sparkContext.hadoopConfiguration))
+ .setBasePath(basePath.toString).build()
// Get list of partitions to delete
val partitionsToDelete = if (parameters.contains(DataSourceWriteOptions.PARTITIONS_TO_DELETE.key())) {
val partitionColsToDelete = parameters(DataSourceWriteOptions.PARTITIONS_TO_DELETE.key()).split(",")
java.util.Arrays.asList(resolvePartitionWildcards(java.util.Arrays.asList(partitionColsToDelete: _*).asScala.toList, jsc,
- hoodieConfig, basePath.toString): _*)
+ tableMetaClient.getStorage, hoodieConfig, basePath.toString): _*)
} else {
val genericRecords = HoodieSparkUtils.createRdd(df, avroRecordName, avroRecordNamespace)
genericRecords.map(gr => keyGenerator.getKey(gr).getPartitionPath).toJavaRDD().distinct().collect()
}
- // Create a HoodieWriteClient & issue the delete.
- val tableMetaClient = HoodieTableMetaClient.builder
- .setConf(HadoopFSUtils.getStorageConfWithCopy(sparkContext.hadoopConfiguration))
- .setBasePath(basePath.toString).build()
+ // Issue the delete.
val schemaStr = new TableSchemaResolver(tableMetaClient).getTableAvroSchema.toString
val client = hoodieWriteClient.getOrElse(DataSourceUtils.createHoodieClient(jsc,
schemaStr, path, tblName,
@@ -564,17 +565,19 @@ class HoodieSparkSqlWriterInternal {
*
* @param partitions list of partitions that may contain wildcards
* @param jsc instance of java spark context
+ * @param storage [[HoodieStorage]] instance
* @param cfg hoodie config
* @param basePath base path
* @return Pair of(boolean, table schema), where first entry will be true only if schema conversion is required.
*/
- private def resolvePartitionWildcards(partitions: List[String], jsc: JavaSparkContext, cfg: HoodieConfig, basePath: String): List[String] = {
+ private def resolvePartitionWildcards(partitions: List[String], jsc: JavaSparkContext,
+ storage: HoodieStorage, cfg: HoodieConfig, basePath: String): List[String] = {
//find out if any of the input partitions have wildcards
//note:spark-sql may url-encode special characters (* -> %2A)
var (wildcardPartitions, fullPartitions) = partitions.partition(partition => partition.matches(".*(\\*|%2A).*"))
val allPartitions = FSUtils.getAllPartitionPaths(new HoodieSparkEngineContext(jsc): HoodieEngineContext,
- HoodieMetadataConfig.newBuilder().fromProperties(cfg.getProps).build(), basePath)
+ storage, HoodieMetadataConfig.newBuilder().fromProperties(cfg.getProps).build(), basePath)
if (fullPartitions.nonEmpty) {
fullPartitions = fullPartitions.filter(partition => allPartitions.contains(partition))
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala
index de0fb8bbe03..52e40669999 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala
@@ -353,7 +353,7 @@ object LogFileIterator extends SparkAdapterSupport {
.enable(true).build()
val dataTableBasePath = getDataTableBasePathFromMetadataTable(tablePath)
val metadataTable = new HoodieBackedTableMetadata(
- new HoodieLocalEngineContext(HadoopFSUtils.getStorageConf(hadoopConf)), metadataConfig,
+ new HoodieLocalEngineContext(HadoopFSUtils.getStorageConf(hadoopConf)), storage, metadataConfig,
dataTableBasePath)
// We have to force full-scan for the MT log record reader, to make sure
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkBaseIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkBaseIndexSupport.scala
index d82ed6ca6d5..2614e4bb37b 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkBaseIndexSupport.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkBaseIndexSupport.scala
@@ -23,10 +23,11 @@ import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.model.FileSlice
import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.metadata.{HoodieMetadataPayload, HoodieTableMetadata}
+
import org.apache.spark.api.java.JavaSparkContext
-import org.apache.spark.sql.{Column, DataFrame, SparkSession}
import org.apache.spark.sql.catalyst.expressions.{And, Expression}
import org.apache.spark.sql.hudi.DataSkippingUtils.translateIntoColumnStatsIndexFilterExpr
+import org.apache.spark.sql.{Column, DataFrame, SparkSession}
import scala.collection.JavaConverters._
@@ -35,7 +36,7 @@ abstract class SparkBaseIndexSupport(spark: SparkSession,
metaClient: HoodieTableMetaClient) {
@transient protected lazy val engineCtx = new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext))
@transient protected lazy val metadataTable: HoodieTableMetadata =
- HoodieTableMetadata.create(engineCtx, metadataConfig, metaClient.getBasePathV2.toString)
+ HoodieTableMetadata.create(engineCtx, metaClient.getStorage, metadataConfig, metaClient.getBasePathV2.toString)
def getIndexName: String
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
index 04a3981d541..2867078aadb 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
@@ -172,7 +172,7 @@ class HoodieCatalogTable(val spark: SparkSession, var table: CatalogTable) exten
def getPartitionPaths: Seq[String] = {
val droppedPartitions = TimelineUtils.getDroppedPartitions(metaClient, org.apache.hudi.common.util.Option.empty(), org.apache.hudi.common.util.Option.empty())
- getAllPartitionPaths(spark, table)
+ getAllPartitionPaths(spark, table, metaClient.getStorage)
.filter(!droppedPartitions.contains(_))
}
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala
index 205620a89a0..7d1171987da 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala
@@ -28,8 +28,8 @@ import org.apache.hudi.common.table.read.HoodieFileGroupReader
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
import org.apache.hudi.common.util.FileIOUtils
import org.apache.hudi.common.util.collection.ExternalSpillableMap.DiskMapType
-import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration
import org.apache.hudi.storage.StorageConfiguration
+import org.apache.hudi.storage.hadoop.{HadoopStorageConfiguration, HoodieHadoopStorage}
import org.apache.hudi.{AvroConversionUtils, HoodieFileIndex, HoodiePartitionCDCFileGroupMapping, HoodiePartitionFileSliceMapping, HoodieSparkUtils, HoodieTableSchema, HoodieTableState, SparkAdapterSupport, SparkFileFormatInternalRowReaderContext}
import org.apache.hadoop.conf.Configuration
@@ -149,7 +149,7 @@ class HoodieFileGroupReaderBasedParquetFileFormat(tableState: HoodieTableState,
.builder().setConf(storageConf).setBasePath(tableState.tablePath).build
val reader = new HoodieFileGroupReader[InternalRow](
readerContext,
- storageConf,
+ new HoodieHadoopStorage(metaClient.getBasePathV2, storageConf),
tableState.tablePath,
tableState.latestCommitTimestamp.get,
fileSlice,
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
index 039760866f3..dc2f5c85f04 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
@@ -17,29 +17,28 @@
package org.apache.spark.sql.hudi
-import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, SparkAdapterSupport}
import org.apache.hudi.client.common.HoodieSparkEngineContext
-import org.apache.hudi.common.config.{DFSPropertiesConfiguration, HoodieMetadataConfig, TypedProperties}
+import org.apache.hudi.common.config.{HoodieMetadataConfig, TypedProperties}
+import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.model.HoodieRecord
-import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
-import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieInstantTimeGenerator, HoodieTimeline}
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline.parseDateFromInstantTime
+import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieInstantTimeGenerator, HoodieTimeline}
+import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
import org.apache.hudi.common.util.PartitionPathEncodeUtils
import org.apache.hudi.exception.HoodieException
+import org.apache.hudi.storage.{HoodieStorage, StoragePathInfo}
import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, SparkAdapterSupport}
-import org.apache.hudi.common.fs.FSUtils
-import org.apache.hudi.storage.StoragePathInfo
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark.api.java.JavaSparkContext
-import org.apache.spark.sql.{AnalysisException, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.Resolver
import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HoodieCatalogTable}
import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression, Literal}
import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
import org.apache.spark.sql.types._
+import org.apache.spark.sql.{AnalysisException, SparkSession}
import java.net.URI
import java.text.SimpleDateFormat
@@ -66,24 +65,25 @@ object HoodieSqlCommonUtils extends SparkAdapterSupport {
avroSchema.map(AvroConversionUtils.convertAvroSchemaToStructType)
}
- def getAllPartitionPaths(spark: SparkSession, table: CatalogTable): Seq[String] = {
+ def getAllPartitionPaths(spark: SparkSession, table: CatalogTable, storage: HoodieStorage): Seq[String] = {
val sparkEngine = new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext))
val metadataConfig = {
val properties = TypedProperties.fromMap((spark.sessionState.conf.getAllConfs ++ table.storage.properties ++ table.properties).asJava)
HoodieMetadataConfig.newBuilder.fromProperties(properties).build()
}
- FSUtils.getAllPartitionPaths(sparkEngine, metadataConfig, getTableLocation(table, spark)).asScala.toSeq
+ FSUtils.getAllPartitionPaths(sparkEngine, storage, metadataConfig, getTableLocation(table, spark)).asScala.toSeq
}
def getFilesInPartitions(spark: SparkSession,
table: CatalogTable,
+ storage: HoodieStorage,
partitionPaths: Seq[String]): Map[String, Seq[StoragePathInfo]] = {
val sparkEngine = new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext))
val metadataConfig = {
val properties = TypedProperties.fromMap((spark.sessionState.conf.getAllConfs ++ table.storage.properties ++ table.properties).asJava)
HoodieMetadataConfig.newBuilder.fromProperties(properties).build()
}
- FSUtils.getFilesInPartitions(sparkEngine, metadataConfig, getTableLocation(table, spark),
+ FSUtils.getFilesInPartitions(sparkEngine, storage, metadataConfig, getTableLocation(table, spark),
partitionPaths.toArray).asScala
.map(e => (e._1, e._2.asScala.toSeq))
.toMap
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala
index 8dc61c32531..53d131d78a5 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala
@@ -17,17 +17,21 @@
package org.apache.spark.sql.hudi.command
+import org.apache.hudi.hadoop.fs.HadoopFSUtils
+import org.apache.hudi.storage.HoodieStorageUtils
+
import org.apache.hadoop.fs.Path
-import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.execution.command.PartitionStatistics
import org.apache.spark.sql.hudi.HoodieSqlCommonUtils
+import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
import org.apache.spark.util.ThreadUtils
import java.util.concurrent.TimeUnit.MILLISECONDS
+import scala.language.postfixOps
import scala.util.control.NonFatal
/**
@@ -81,9 +85,11 @@ case class RepairHoodieTableCommand(tableName: TableIdentifier,
} else 0
val addedAmount = if (enableAddPartitions) {
val total = partitionSpecsAndLocs.length
- val partitionStats = if (spark.sqlContext.conf.gatherFastStats) {
- HoodieSqlCommonUtils.getFilesInPartitions(spark, table, partitionSpecsAndLocs
- .map(_._2.toString))
+ val partitionList = partitionSpecsAndLocs.map(_._2.toString)
+ val partitionStats = if (spark.sqlContext.conf.gatherFastStats && total > 0) {
+ HoodieSqlCommonUtils.getFilesInPartitions(spark, table,
+ HoodieStorageUtils.getStorage(partitionList.head, HadoopFSUtils.getStorageConf(spark.sessionState.newHadoopConf())),
+ partitionList)
.mapValues(statuses => PartitionStatistics(statuses.length, statuses.map(_.getLength).sum))
} else {
Map.empty[String, PartitionStatistics]
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala
index 4afe609a570..57eefbfa5e0 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala
@@ -26,6 +26,7 @@ import org.apache.hudi.common.table.timeline.TimelineUtils.{HollowCommitHandling
import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
import org.apache.hudi.common.util.TablePathUtils
import org.apache.hudi.hadoop.fs.HadoopFSUtils
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hudi.storage.{HoodieStorageUtils, StoragePath}
import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, IncrementalRelation, MergeOnReadIncrementalRelation, SparkAdapterSupport}
@@ -58,7 +59,7 @@ class HoodieStreamSource(
private lazy val tablePath: StoragePath = {
val path = new StoragePath(parameters.getOrElse("path", "Missing 'path' option"))
- val fs = HoodieStorageUtils.getStorage(path, storageConf)
+ val fs = new HoodieHadoopStorage(path, storageConf)
TablePathUtils.getTablePath(fs, path).get()
}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala
index c2a717e2764..e534a13d766 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala
@@ -48,7 +48,7 @@ object SparkHelpers {
sourceFile: StoragePath,
destinationFile: StoragePath,
keysToSkip: Set[String]) {
- val sourceRecords = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET).readAvroRecords(conf, sourceFile).asScala
+ val sourceRecords = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET).readAvroRecords(storage, sourceFile).asScala
val schema: Schema = sourceRecords.head.getSchema
val filter: BloomFilter = BloomFilterFactory.createBloomFilter(
BLOOM_FILTER_NUM_ENTRIES_VALUE.defaultValue.toInt, BLOOM_FILTER_FPP_VALUE.defaultValue.toDouble,
@@ -134,13 +134,13 @@ class SparkHelper(sqlContext: SQLContext, fs: FileSystem) {
* Checks that all the keys in the file, have been added to the bloom filter
* in the footer
*
- * @param conf
- * @param sqlContext
- * @param file
- * @return
+ * @param storage [[HoodieStorage]] instance.
+ * @param sqlContext Spark SQL context.
+ * @param file File path.
+ * @return <pre>true</pre> if all keys are added to the bloom filter; <pre>false</pre> otherwise.
*/
- def fileKeysAgainstBF(conf: StorageConfiguration[_], sqlContext: SQLContext, file: String): Boolean = {
- val bf = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET).readBloomFilterFromMetadata(conf, new StoragePath(file))
+ def fileKeysAgainstBF(storage: HoodieStorage, sqlContext: SQLContext, file: String): Boolean = {
+ val bf = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET).readBloomFilterFromMetadata(storage, new StoragePath(file))
val foundCount = sqlContext.parquetFile(file)
.select(s"`${HoodieRecord.RECORD_KEY_METADATA_FIELD}`")
.collect().count(r => !bf.mightContain(r.getString(0)))
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
index deb62b607f3..b47f434e609 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
@@ -33,6 +33,7 @@ import java.io.IOException
import java.util
import java.util.Properties
import java.util.function.{Consumer, Supplier}
+
import scala.collection.JavaConverters._
class RepairMigratePartitionMetaProcedure extends BaseProcedure with ProcedureBuilder with Logging {
@@ -62,7 +63,7 @@ class RepairMigratePartitionMetaProcedure extends BaseProcedure with ProcedureBu
val metaClient = createMetaClient(jsc, tablePath)
val engineContext: HoodieLocalEngineContext = new HoodieLocalEngineContext(metaClient.getStorageConf)
- val partitionPaths: util.List[String] = FSUtils.getAllPartitionPaths(engineContext, tablePath, false)
+ val partitionPaths: util.List[String] = FSUtils.getAllPartitionPaths(engineContext, metaClient.getStorage, tablePath, false)
val basePath: StoragePath = new StoragePath(tablePath)
val rows = new util.ArrayList[Row](partitionPaths.size)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowColumnStatsOverlapProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowColumnStatsOverlapProcedure.scala
index 632d72a0a42..9d2583392ae 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowColumnStatsOverlapProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowColumnStatsOverlapProcedure.scala
@@ -38,8 +38,9 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
import java.util.function.{Function, Supplier}
-import scala.collection.mutable
+
import scala.collection.JavaConverters._
+import scala.collection.mutable
/**
* Calculate the degree of overlap between column stats.
@@ -108,7 +109,7 @@ class ShowColumnStatsOverlapProcedure extends BaseProcedure with ProcedureBuilde
val columnStatsIndex = new ColumnStatsIndexSupport(spark, schema, metadataConfig, metaClient)
val fsView = buildFileSystemView(table)
val engineCtx = new HoodieSparkEngineContext(jsc)
- val metaTable = HoodieTableMetadata.create(engineCtx, metadataConfig, basePath)
+ val metaTable = HoodieTableMetadata.create(engineCtx, metaClient.getStorage, metadataConfig, basePath)
val allFileSlices = getAllFileSlices(partitionsSeq, metaTable, fsView)
val fileSlicesSizeByPartition = allFileSlices.groupBy(_.getPartitionPath).mapValues(_.size)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala
index a30c36b2f01..43aaa48f82b 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.client.common.HoodieSparkEngineContext
import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.hadoop.fs.HadoopFSUtils
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.fs.Path
import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS
@@ -49,9 +50,11 @@ class ShowInvalidParquetProcedure extends BaseProcedure with ProcedureBuilder {
val srcPath = getArgValueOrDefault(args, PARAMETERS(0)).get.asInstanceOf[String]
val limit = getArgValueOrDefault(args, PARAMETERS(1))
- val partitionPaths: java.util.List[String] = FSUtils.getAllPartitionPaths(new HoodieSparkEngineContext(jsc), srcPath, false)
- val javaRdd: JavaRDD[String] = jsc.parallelize(partitionPaths, partitionPaths.size())
val storageConf = HadoopFSUtils.getStorageConfWithCopy(jsc.hadoopConfiguration())
+ val storage = new HoodieHadoopStorage(srcPath, storageConf)
+ val partitionPaths: java.util.List[String] = FSUtils.getAllPartitionPaths(
+ new HoodieSparkEngineContext(jsc), storage, srcPath, false)
+ val javaRdd: JavaRDD[String] = jsc.parallelize(partitionPaths, partitionPaths.size())
val parquetRdd = javaRdd.rdd.map(part => {
val fs = HadoopFSUtils.getFs(new Path(srcPath), storageConf.unwrap())
HadoopFSUtils.getAllDataFilesInPartition(fs, HadoopFSUtils.constructAbsolutePathInHadoopPath(srcPath, part))
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableColumnStatsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableColumnStatsProcedure.scala
index 5b77515782c..c60b2538bb6 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableColumnStatsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableColumnStatsProcedure.scala
@@ -38,8 +38,9 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
import java.util.function.{Function, Supplier}
-import scala.collection.mutable
+
import scala.collection.JavaConverters._
+import scala.collection.mutable
class ShowMetadataTableColumnStatsProcedure extends BaseProcedure with ProcedureBuilder with Logging {
private val PARAMETERS = Array[ProcedureParameter](
@@ -82,7 +83,7 @@ class ShowMetadataTableColumnStatsProcedure extends BaseProcedure with Procedure
val allFileSlices: Set[FileSlice] = {
if (partitionsSeq.isEmpty) {
val engineCtx = new HoodieSparkEngineContext(jsc)
- val metaTable = HoodieTableMetadata.create(engineCtx, metadataConfig, basePath)
+ val metaTable = HoodieTableMetadata.create(engineCtx, metaClient.getStorage, metadataConfig, basePath)
metaTable.getAllPartitionPaths
.asScala
.flatMap(path => fsView.getLatestFileSlices(path).iterator().asScala)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala
index c694f8c65e9..c6b52dd3e04 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala
@@ -21,7 +21,6 @@ import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.engine.HoodieLocalEngineContext
import org.apache.hudi.common.util.{HoodieTimer, StringUtils}
import org.apache.hudi.exception.HoodieException
-import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.metadata.HoodieBackedTableMetadata
import org.apache.hudi.storage.{StoragePath, StoragePathInfo}
@@ -59,7 +58,8 @@ class ShowMetadataTableFilesProcedure() extends BaseProcedure with ProcedureBuil
val basePath = getBasePath(table)
val metaClient = createMetaClient(jsc, basePath)
val config = HoodieMetadataConfig.newBuilder.enable(true).build
- val metaReader = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getStorageConf), config, basePath)
+ val metaReader = new HoodieBackedTableMetadata(
+ new HoodieLocalEngineContext(metaClient.getStorageConf), metaClient.getStorage, config, basePath)
if (!metaReader.enabled){
throw new HoodieException(s"Metadata Table not enabled/initialized.")
}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala
index 9a7867d6648..9f0e23a86fe 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala
@@ -21,7 +21,10 @@ import org.apache.hudi.client.common.HoodieSparkEngineContext
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.util.HoodieTimer
import org.apache.hudi.exception.HoodieException
+import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.metadata.HoodieBackedTableMetadata
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
+
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -29,6 +32,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
import java.util.Collections
import java.util.function.Supplier
+
import scala.collection.JavaConverters.asScalaIteratorConverter
class ShowMetadataTablePartitionsProcedure() extends BaseProcedure with ProcedureBuilder with Logging {
@@ -50,8 +54,9 @@ class ShowMetadataTablePartitionsProcedure() extends BaseProcedure with Procedur
val table = getArgValueOrDefault(args, PARAMETERS(0))
val basePath = getBasePath(table)
+ val storage = new HoodieHadoopStorage(basePath, spark.sessionState.newHadoopConf())
val config = HoodieMetadataConfig.newBuilder.enable(true).build
- val metadata = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), config, basePath)
+ val metadata = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), storage, config, basePath)
if (!metadata.enabled){
throw new HoodieException(s"Metadata Table not enabled/initialized.")
}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala
index f67c6ac5406..37e6c92fa23 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
import java.util.function.Supplier
+
import scala.collection.JavaConverters._
class ShowMetadataTableStatsProcedure() extends BaseProcedure with ProcedureBuilder {
@@ -50,7 +51,8 @@ class ShowMetadataTableStatsProcedure() extends BaseProcedure with ProcedureBuil
val basePath = getBasePath(table)
val metaClient = createMetaClient(jsc, basePath)
val config = HoodieMetadataConfig.newBuilder.enable(true).build
- val metadata = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getStorageConf), config, basePath)
+ val metadata = new HoodieBackedTableMetadata(
+ new HoodieLocalEngineContext(metaClient.getStorageConf), metaClient.getStorage, config, basePath)
val stats = metadata.stats
val rows = new util.ArrayList[Row]
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala
index 61157bb9253..c1d79a768fa 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala
@@ -21,7 +21,6 @@ import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.engine.HoodieLocalEngineContext
import org.apache.hudi.common.util.HoodieTimer
import org.apache.hudi.exception.HoodieException
-import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.metadata.HoodieBackedTableMetadata
import org.apache.hudi.storage.{StoragePath, StoragePathInfo}
@@ -64,7 +63,7 @@ class ValidateMetadataTableFilesProcedure() extends BaseProcedure with Procedure
val metaClient = createMetaClient(jsc, basePath)
val config = HoodieMetadataConfig.newBuilder.enable(true).build
val metadataReader = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getStorageConf),
- config, basePath)
+ metaClient.getStorage, config, basePath)
if (!metadataReader.enabled) {
throw new HoodieException(s"Metadata Table not enabled/initialized.")
@@ -72,7 +71,7 @@ class ValidateMetadataTableFilesProcedure() extends BaseProcedure with Procedure
val fsConfig = HoodieMetadataConfig.newBuilder.enable(false).build
val fsMetaReader = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getStorageConf),
- fsConfig, basePath)
+ metaClient.getStorage, fsConfig, basePath)
val timer = HoodieTimer.start
val metadataPartitions = metadataReader.getAllPartitionPaths
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/ColumnStatsIndexHelper.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/ColumnStatsIndexHelper.java
index 8ff46be7621..6653c9cf969 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/ColumnStatsIndexHelper.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/ColumnStatsIndexHelper.java
@@ -24,8 +24,10 @@ import org.apache.hudi.common.util.ParquetUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.JavaScalaConverters;
import org.apache.spark.SparkContext;
@@ -177,13 +179,15 @@ public class ColumnStatsIndexHelper {
ParquetUtils utils = (ParquetUtils) FileFormatUtils.getInstance(HoodieFileFormat.PARQUET);
Iterable<String> iterable = () -> paths;
return StreamSupport.stream(iterable.spliterator(), false)
- .flatMap(path ->
- utils.readColumnStatsFromMetadata(
- storageConf,
- new StoragePath(path),
- columnNames
- )
- .stream()
+ .flatMap(path -> {
+ HoodieStorage storage = new HoodieHadoopStorage(path, storageConf);
+ return utils.readColumnStatsFromMetadata(
+ storage,
+ new StoragePath(path),
+ columnNames
+ )
+ .stream();
+ }
)
.iterator();
})
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java
index ae617a7cb91..b6795bc2a2a 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java
@@ -181,7 +181,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
srcPath, context).stream().findAny().map(p -> p.getValue().stream().findAny())
.orElse(null).get().getPath()).toString();
HoodieAvroParquetReader parquetReader =
- new HoodieAvroParquetReader(metaClient.getStorageConf(), new StoragePath(filePath));
+ new HoodieAvroParquetReader(metaClient.getStorage(), new StoragePath(filePath));
return parquetReader.getSchema();
}
@@ -413,7 +413,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
reloadInputFormats();
List<GenericRecord> records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, roJobConf, false, schema, TRIP_HIVE_COLUMN_TYPES, false, new ArrayList<>());
assertEquals(totalRecords, records.size());
@@ -431,7 +431,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
seenKeys = new HashSet<>();
records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, rtJobConf, true, schema, TRIP_HIVE_COLUMN_TYPES, false, new ArrayList<>());
assertEquals(totalRecords, records.size());
@@ -447,7 +447,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
reloadInputFormats();
records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, roJobConf, false, schema, TRIP_HIVE_COLUMN_TYPES,
true, HoodieRecord.HOODIE_META_COLUMNS);
@@ -464,7 +464,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
seenKeys = new HashSet<>();
records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, rtJobConf, true, schema, TRIP_HIVE_COLUMN_TYPES, true,
HoodieRecord.HOODIE_META_COLUMNS);
@@ -479,7 +479,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
reloadInputFormats();
records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, roJobConf, false, schema, TRIP_HIVE_COLUMN_TYPES, true,
Arrays.asList("_row_key"));
@@ -496,7 +496,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
seenKeys = new HashSet<>();
records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, rtJobConf, true, schema, TRIP_HIVE_COLUMN_TYPES, true,
Arrays.asList("_row_key"));
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java
index c9eacc1bd60..723d2389d22 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java
@@ -188,12 +188,12 @@ public class TestSparkConsistentBucketClustering extends HoodieSparkClientTestHa
StoragePath metadataPath =
FSUtils.constructAbsolutePath(table.getMetaClient().getHashingMetadataPath(), p);
try {
- table.getMetaClient().getStorage().listDirectEntries(metadataPath).forEach(fl -> {
+ table.getStorage().listDirectEntries(metadataPath).forEach(fl -> {
if (fl.getPath().getName()
.contains(HoodieConsistentHashingMetadata.HASHING_METADATA_COMMIT_FILE_SUFFIX)) {
try {
// delete commit marker to test recovery job
- table.getMetaClient().getStorage().deleteDirectory(fl.getPath());
+ table.getStorage().deleteDirectory(fl.getPath());
} catch (IOException e) {
throw new RuntimeException(e);
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
index 95f151336c7..06829365cad 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
@@ -111,7 +111,7 @@ public class TestHoodieInternalRowParquetWriter extends HoodieSparkClientTestHar
String minKey = recordKeys.stream().min(Comparator.naturalOrder()).get();
String maxKey = recordKeys.stream().max(Comparator.naturalOrder()).get();
- FileMetaData parquetMetadata = ParquetUtils.readMetadata(storageConf, filePath).getFileMetaData();
+ FileMetaData parquetMetadata = ParquetUtils.readMetadata(storage, filePath).getFileMetaData();
Map<String, String> extraMetadata = parquetMetadata.getKeyValueMetaData();
@@ -120,7 +120,7 @@ public class TestHoodieInternalRowParquetWriter extends HoodieSparkClientTestHar
assertEquals(extraMetadata.get(HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE), BloomFilterTypeCode.DYNAMIC_V0.name());
// Step 3: Make sure Bloom Filter contains all the record keys
- BloomFilter bloomFilter = new ParquetUtils().readBloomFilterFromMetadata(storageConf, filePath);
+ BloomFilter bloomFilter = new ParquetUtils().readBloomFilterFromMetadata(storage, filePath);
recordKeys.forEach(recordKey -> {
assertTrue(bloomFilter.mightContain(recordKey));
});
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala
index 75af17fe48d..57e172bca5b 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala
@@ -602,6 +602,7 @@ class TestHoodieFileIndex extends HoodieSparkClientTestBase with ScalaAssertionS
// Test getting partition paths in a subset of directories
val metadata = HoodieTableMetadata.create(context,
+ metaClient.getStorage,
HoodieMetadataConfig.newBuilder().enable(enableMetadataTable).build(),
metaClient.getBasePathV2.toString)
assertEquals(
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
index 6a746230c4a..fa53b908bd0 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
@@ -177,7 +177,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
assertTrue(snapshot0.filter("_hoodie_partition_path = '" + HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH + "'").count() > 0)
assertTrue(snapshot0.filter("_hoodie_partition_path = '" + HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH + "'").count() > 0)
assertTrue(snapshot0.filter("_hoodie_partition_path = '" + HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH + "'").count() > 0)
- val storage = HoodieStorageUtils.getStorage(new StoragePath(basePath), HoodieTestUtils.getDefaultStorageConf)
+ val storage = HoodieTestUtils.getStorage(new StoragePath(basePath))
assertTrue(storage.exists(new StoragePath(basePath + "/" + HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)))
assertTrue(storage.exists(new StoragePath(basePath + "/" + HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH)))
assertTrue(storage.exists(new StoragePath(basePath + "/" + HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH)))
@@ -202,7 +202,8 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
snapshot1.cache()
assertEquals(300, snapshot1.count())
- var partitionPaths = FSUtils.getAllPartitionPaths(new HoodieSparkEngineContext(jsc), HoodieMetadataConfig.newBuilder().build(), basePath)
+ var partitionPaths = FSUtils.getAllPartitionPaths(
+ new HoodieSparkEngineContext(jsc), storage, HoodieMetadataConfig.newBuilder().build(), basePath)
assertTrue(partitionPaths.contains("100/rider-123"))
assertTrue(partitionPaths.contains("200/rider-456"))
@@ -225,7 +226,8 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.mode(SaveMode.Overwrite)
.save(basePath)
- partitionPaths = FSUtils.getAllPartitionPaths(new HoodieSparkEngineContext(jsc), HoodieMetadataConfig.newBuilder().build(), basePath)
+ partitionPaths = FSUtils.getAllPartitionPaths(
+ new HoodieSparkEngineContext(jsc), storage, HoodieMetadataConfig.newBuilder().build(), basePath)
assertEquals(partitionPaths.size(), 1)
assertEquals(partitionPaths.get(0), "")
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala
index 953ab6ae1da..f06c8e3c330 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala
@@ -31,6 +31,7 @@ import org.apache.hudi.functional.ColumnStatIndexTestBase.ColumnStatsTestCase
import org.apache.hudi.storage.StoragePath
import org.apache.hudi.{ColumnStatsIndexSupport, DataSourceWriteOptions}
+import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
@@ -455,15 +456,14 @@ class TestColumnStatsIndex extends ColumnStatIndexTestBase {
val utils = new ParquetUtils
- val conf = HoodieTestUtils.getDefaultStorageConf
val path = new Path(pathStr)
- val fs = path.getFileSystem(conf.unwrap)
+ val storage = HoodieTestUtils.getStorage(new StoragePath(pathStr))
+ val fs = path.getFileSystem(storage.getConf.unwrapAs(classOf[Configuration]))
val parquetFilePath = new StoragePath(
fs.listStatus(path).filter(fs => fs.getPath.getName.endsWith(".parquet")).toSeq.head.getPath.toUri)
- val ranges = utils.readColumnStatsFromMetadata(conf, parquetFilePath,
- Seq("c1", "c2", "c3a", "c3b", "c3c", "c4", "c5", "c6", "c7", "c8").asJava)
+ val ranges = utils.readColumnStatsFromMetadata(storage, parquetFilePath, Seq("c1", "c2", "c3a", "c3b", "c3c", "c4", "c5", "c6", "c7", "c8").asJava)
ranges.asScala.foreach(r => {
// NOTE: Unfortunately Parquet can't compute statistics for Timestamp column, hence we
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkDataSource.scala
index 8c7e01488fc..04b37256a01 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkDataSource.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkDataSource.scala
@@ -30,6 +30,7 @@ import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.metadata.{HoodieBackedTableMetadata, HoodieTableMetadata}
import org.apache.hudi.storage.StoragePath
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness.getSparkSqlConf
@@ -137,7 +138,8 @@ class TestMetadataTableWithSparkDataSource extends SparkClientFunctionalTestHarn
val partitionPathToTest = "2015/03/16"
val engineContext = new HoodieSparkEngineContext(jsc())
val metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexColumnStats(true).build();
- val baseTableMetada: HoodieTableMetadata = new HoodieBackedTableMetadata(engineContext, metadataConfig, s"$basePath", false)
+ val baseTableMetada: HoodieTableMetadata = new HoodieBackedTableMetadata(
+ engineContext, hoodieStorage(), metadataConfig, s"$basePath", false)
val fileStatuses = baseTableMetada.getAllFilesInPartition(new StoragePath(s"$basePath/" + partitionPathToTest))
val fileName = fileStatuses.get(0).getPath.getName
@@ -151,7 +153,8 @@ class TestMetadataTableWithSparkDataSource extends SparkClientFunctionalTestHarn
// read parquet file and verify stats
val colRangeMetadataList: java.util.List[HoodieColumnRangeMetadata[Comparable[_]]] = new ParquetUtils()
- .readColumnStatsFromMetadata(HadoopFSUtils.getStorageConf(jsc().hadoopConfiguration()),
+ .readColumnStatsFromMetadata(
+ new HoodieHadoopStorage(fileStatuses.get(0).getPath, HadoopFSUtils.getStorageConf(jsc().hadoopConfiguration())),
fileStatuses.get(0).getPath, Collections.singletonList("begin_lat"))
val columnRangeMetadata = colRangeMetadataList.get(0)
@@ -189,7 +192,8 @@ class TestMetadataTableWithSparkDataSource extends SparkClientFunctionalTestHarn
val partitionPathToTest = ""
val engineContext = new HoodieSparkEngineContext(jsc())
val metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexColumnStats(true).build();
- val baseTableMetada: HoodieTableMetadata = new HoodieBackedTableMetadata(engineContext, metadataConfig, s"$basePath", false)
+ val baseTableMetada: HoodieTableMetadata = new HoodieBackedTableMetadata(
+ engineContext, hoodieStorage(), metadataConfig, s"$basePath", false)
val allPartitionPaths = baseTableMetada.getAllPartitionPaths
assertEquals(allPartitionPaths.size(), 1)
@@ -207,7 +211,8 @@ class TestMetadataTableWithSparkDataSource extends SparkClientFunctionalTestHarn
// read parquet file and verify stats
val colRangeMetadataList: java.util.List[HoodieColumnRangeMetadata[Comparable[_]]] = new ParquetUtils()
- .readColumnStatsFromMetadata(HadoopFSUtils.getStorageConf(jsc().hadoopConfiguration()),
+ .readColumnStatsFromMetadata(
+ new HoodieHadoopStorage(fileStatuses.get(0).getPath, HadoopFSUtils.getStorageConf(jsc().hadoopConfiguration())),
fileStatuses.get(0).getPath, Collections.singletonList("begin_lat"))
val columnRangeMetadata = colRangeMetadataList.get(0)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
index 8c633a25aab..62c2508e63a 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
@@ -37,6 +37,7 @@ import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
import java.util.function.Predicate
import java.util.{Collections, List, Optional}
+
import scala.collection.JavaConverters._
class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
@@ -388,7 +389,7 @@ class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
new SerializableFunctionUnchecked[HoodieTableMetaClient, HoodieTableMetadata] {
override def apply(v1: HoodieTableMetaClient): HoodieTableMetadata = {
HoodieTableMetadata.create(
- engineContext, metadataConfig, metaClient.getBasePathV2.toString)
+ engineContext, metaClient.getStorage, metadataConfig, metaClient.getBasePathV2.toString)
}
}
)
diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParquetSchemaEvolutionUtils.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParquetSchemaEvolutionUtils.scala
index bd3bb1545d1..967c647b284 100644
--- a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParquetSchemaEvolutionUtils.scala
+++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParquetSchemaEvolutionUtils.scala
@@ -18,8 +18,6 @@
package org.apache.spark.sql.execution.datasources
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
import org.apache.hudi.SparkAdapterSupport
import org.apache.hudi.client.utils.SparkInternalSchemaConverter
import org.apache.hudi.common.fs.FSUtils
@@ -31,7 +29,11 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.StoragePath
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
import org.apache.parquet.hadoop.metadata.FileMetaData
import org.apache.spark.sql.HoodieSchemaUtils
import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
@@ -60,8 +62,8 @@ abstract class Spark3ParquetSchemaEvolutionUtils(sharedConf: Configuration,
private lazy val fileSchema: InternalSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
- InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, HadoopFSUtils.getStorageConf(sharedConf), if (validCommits == null) "" else validCommits)
+ InternalSchemaCache.getInternalSchemaByVersionId(commitInstantTime, tablePath,
+ new HoodieHadoopStorage(tablePath, sharedConf), if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala
index f672f3068c3..bf6e222b763 100644
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
@@ -159,8 +160,9 @@ class Spark30LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala
index 74c75b0024d..aa1b798241c 100644
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
@@ -159,8 +160,9 @@ class Spark31LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32LegacyHoodieParquetFileFormat.scala
index 68f8ad2e30b..44d420c7501 100644
--- a/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32LegacyHoodieParquetFileFormat.scala
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
@@ -158,8 +159,9 @@ class Spark32LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala
index f2946b04d4e..d39d12b3fe2 100644
--- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala
@@ -29,6 +29,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
@@ -160,8 +161,9 @@ class Spark33LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala
index 9347f0024f2..8818cb5672f 100644
--- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapred.FileSplit
@@ -172,8 +173,8 @@ class Spark34LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
- InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
+ InternalSchemaCache.getInternalSchemaByVersionId(commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala
index 4ecdf451031..6286a19f080 100644
--- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapred.FileSplit
@@ -173,8 +174,9 @@ class Spark35LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
index 524c6009d31..9566442e4ec 100644
--- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
+++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
@@ -122,6 +122,7 @@ public abstract class HoodieSyncClient implements HoodieMetaSyncOperations, Auto
public List<String> getAllPartitionPathsOnStorage() {
HoodieLocalEngineContext engineContext = new HoodieLocalEngineContext(metaClient.getStorageConf());
return FSUtils.getAllPartitionPaths(engineContext,
+ metaClient.getStorage(),
config.getString(META_SYNC_BASE_PATH),
config.getBoolean(META_SYNC_USE_FILE_LISTING_FROM_METADATA));
}
diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
index eb52fe4586d..5386805e8df 100644
--- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
+++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
@@ -104,7 +104,7 @@ public class ManifestFileWriter {
@VisibleForTesting
static Stream<String> getLatestBaseFiles(boolean canUseMetadataTable, HoodieEngineContext engContext, HoodieTableMetaClient metaClient,
boolean useAbsolutePath) {
- List<String> partitions = FSUtils.getAllPartitionPaths(engContext, metaClient.getBasePath(), canUseMetadataTable);
+ List<String> partitions = FSUtils.getAllPartitionPaths(engContext, metaClient.getStorage(), metaClient.getBasePath(), canUseMetadataTable);
LOG.info("Retrieve all partitions: " + partitions.size());
HoodieTableFileSystemView fsView = null;
try {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
index 662929c5015..a2c578b1143 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
@@ -300,7 +300,7 @@ public class HoodieDataTableValidator implements Serializable {
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
try {
HoodieTableMetadata tableMetadata = new FileSystemBackedTableMetadata(
- engineContext, metaClient.getTableConfig(), engineContext.getStorageConf(), cfg.basePath);
+ engineContext, metaClient.getTableConfig(), metaClient.getStorage(), cfg.basePath);
List<StoragePath> allDataFilePaths =
HoodieDataTableUtils.getBaseAndLogFilePathsFromFileSystem(tableMetadata, cfg.basePath);
// verify that no data files present with commit time < earliest commit in active timeline.
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
index b9e72ed711d..8cc804cacac 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
@@ -622,7 +622,7 @@ public class HoodieMetadataTableValidator implements Serializable {
List<String> allPartitionPathsFromFS = getPartitionsFromFileSystem(engineContext, basePath, metaClient.getStorage(),
completedTimeline);
- List<String> allPartitionPathsMeta = getPartitionsFromMDT(engineContext, basePath);
+ List<String> allPartitionPathsMeta = getPartitionsFromMDT(engineContext, basePath, metaClient.getStorage());
Collections.sort(allPartitionPathsFromFS);
Collections.sort(allPartitionPathsMeta);
@@ -675,14 +675,15 @@ public class HoodieMetadataTableValidator implements Serializable {
}
@VisibleForTesting
- List<String> getPartitionsFromMDT(HoodieEngineContext engineContext, String basePath) {
- return FSUtils.getAllPartitionPaths(engineContext, basePath, true);
+ List<String> getPartitionsFromMDT(HoodieEngineContext engineContext, String basePath,
+ HoodieStorage storage) {
+ return FSUtils.getAllPartitionPaths(engineContext, storage, basePath, true);
}
@VisibleForTesting
List<String> getPartitionsFromFileSystem(HoodieEngineContext engineContext, String basePath,
HoodieStorage storage, HoodieTimeline completedTimeline) {
- List<String> allPartitionPathsFromFS = FSUtils.getAllPartitionPaths(engineContext, basePath, false);
+ List<String> allPartitionPathsFromFS = FSUtils.getAllPartitionPaths(engineContext, storage, basePath, false);
// ignore partitions created by uncommitted ingestion.
return allPartitionPathsFromFS.stream().parallel().filter(part -> {
@@ -1392,7 +1393,8 @@ public class HoodieMetadataTableValidator implements Serializable {
.build();
this.fileSystemView = FileSystemViewManager.createInMemoryFileSystemView(engineContext,
metaClient, metadataConfig);
- this.tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, metaClient.getBasePathV2().toString());
+ this.tableMetadata = HoodieTableMetadata.create(
+ engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePathV2().toString());
if (metaClient.getCommitsTimeline().filterCompletedInstants().countInstants() > 0) {
this.allColumnNameList = getAllColumnNames();
}
@@ -1439,7 +1441,7 @@ public class HoodieMetadataTableValidator implements Serializable {
} else {
return baseFileNameList.stream().flatMap(filename ->
FileFormatUtils.getInstance(HoodieFileFormat.PARQUET).readColumnStatsFromMetadata(
- metaClient.getStorageConf(),
+ metaClient.getStorage(),
new StoragePath(FSUtils.constructAbsolutePath(metaClient.getBasePathV2(), partitionPath), filename),
allColumnNameList).stream())
.sorted(new HoodieColumnRangeMetadataComparator())
@@ -1487,7 +1489,7 @@ public class HoodieMetadataTableValidator implements Serializable {
HoodieConfig hoodieConfig = new HoodieConfig();
hoodieConfig.setValue(HoodieReaderConfig.USE_NATIVE_HFILE_READER,
Boolean.toString(ConfigUtils.getBooleanWithAltKeys(props, HoodieReaderConfig.USE_NATIVE_HFILE_READER)));
- try (HoodieFileReader fileReader = getHoodieSparkIOFactory(metaClient.getStorageConf())
+ try (HoodieFileReader fileReader = getHoodieSparkIOFactory(metaClient.getStorage())
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, path)) {
bloomFilter = fileReader.readBloomFilter();
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java
index 8494ef4806a..d5e6475c301 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java
@@ -167,7 +167,7 @@ public class HoodieRepairTool {
.build();
this.tableMetadata = new FileSystemBackedTableMetadata(
- context, metaClient.getTableConfig(), context.getStorageConf(), cfg.basePath);
+ context, metaClient.getTableConfig(), metaClient.getStorage(), cfg.basePath);
}
public boolean run() {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
index a5b18c47228..ca3f7ab47b2 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
@@ -36,6 +36,7 @@ import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
@@ -102,7 +103,8 @@ public class HoodieSnapshotCopier implements Serializable {
LOG.info(String.format("Starting to snapshot latest version files which are also no-late-than %s.",
latestCommitTimestamp));
- List<String> partitions = FSUtils.getAllPartitionPaths(context, baseDir, useFileListingFromMetadata);
+ List<String> partitions = FSUtils.getAllPartitionPaths(
+ context, new HoodieHadoopStorage(fs), baseDir, useFileListingFromMetadata);
if (partitions.size() > 0) {
LOG.info(String.format("The job needs to copy %d partitions.", partitions.size()));
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
index aacf7708d42..16013adb051 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
@@ -40,6 +40,7 @@ import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.JavaScalaConverters;
import org.apache.hudi.utilities.exception.HoodieSnapshotExporterException;
@@ -137,7 +138,7 @@ public class HoodieSnapshotExporter {
latestCommitTimestamp));
final HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
- final List<String> partitions = getPartitions(engineContext, cfg);
+ final List<String> partitions = getPartitions(engineContext, cfg, new HoodieHadoopStorage(sourceFs));
if (partitions.isEmpty()) {
throw new HoodieSnapshotExporterException("The source dataset has 0 partition to snapshot.");
}
@@ -160,8 +161,9 @@ public class HoodieSnapshotExporter {
return latestCommit.isPresent() ? Option.of(latestCommit.get().getTimestamp()) : Option.empty();
}
- private List<String> getPartitions(HoodieEngineContext engineContext, Config cfg) {
- return FSUtils.getAllPartitionPaths(engineContext, cfg.sourceBasePath, true);
+ private List<String> getPartitions(HoodieEngineContext engineContext, Config cfg,
+ HoodieStorage storage) {
+ return FSUtils.getAllPartitionPaths(engineContext, storage, cfg.sourceBasePath, true);
}
private void createSuccessTag(FileSystem fs, Config cfg) throws IOException {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
index 1a6a1ba4f82..ff655dfd017 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
@@ -35,6 +35,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
@@ -276,8 +277,9 @@ public class TableSizeStats implements Serializable {
.enable(isMetadataEnabled(basePath, jsc))
.build();
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
- HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, basePath);
StorageConfiguration<?> storageConf = HadoopFSUtils.getStorageConfWithCopy(jsc.hadoopConfiguration());
+ HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(
+ engineContext, new HoodieHadoopStorage(basePath, storageConf), metadataConfig, basePath);
List<String> allPartitions = tableMetadata.getAllPartitionPaths();
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java
index 1dd008da237..9ad99798663 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java
@@ -21,7 +21,7 @@ package org.apache.hudi.utilities.deltastreamer;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.utilities.streamer.HoodieStreamerMetrics;
/**
@@ -31,11 +31,11 @@ import org.apache.hudi.utilities.streamer.HoodieStreamerMetrics;
@Deprecated
public class HoodieDeltaStreamerMetrics extends HoodieStreamerMetrics {
- public HoodieDeltaStreamerMetrics(HoodieWriteConfig writeConfig, StorageConfiguration<?> storageConf) {
- super(writeConfig.getMetricsConfig(), storageConf);
+ public HoodieDeltaStreamerMetrics(HoodieWriteConfig writeConfig, HoodieStorage storage) {
+ super(writeConfig.getMetricsConfig(), storage);
}
- public HoodieDeltaStreamerMetrics(HoodieMetricsConfig metricsConfig, StorageConfiguration<?> storageConf) {
- super(metricsConfig, storageConf);
+ public HoodieDeltaStreamerMetrics(HoodieMetricsConfig metricsConfig, HoodieStorage storage) {
+ super(metricsConfig, storage);
}
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java
index 378ba45e3e9..632f8bbafb3 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java
@@ -20,7 +20,7 @@ package org.apache.hudi.utilities.ingestion;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import com.codahale.metrics.Timer;
@@ -31,17 +31,17 @@ import java.io.Serializable;
*/
public abstract class HoodieIngestionMetrics implements Serializable {
- protected final StorageConfiguration<?> storageConf;
+ protected final HoodieStorage storage;
protected final HoodieMetricsConfig writeConfig;
- public HoodieIngestionMetrics(HoodieWriteConfig writeConfig, StorageConfiguration<?> storageConf) {
- this(writeConfig.getMetricsConfig(), storageConf);
+ public HoodieIngestionMetrics(HoodieWriteConfig writeConfig, HoodieStorage storage) {
+ this(writeConfig.getMetricsConfig(), storage);
}
- public HoodieIngestionMetrics(HoodieMetricsConfig writeConfig, StorageConfiguration<?> storageConf) {
+ public HoodieIngestionMetrics(HoodieMetricsConfig writeConfig, HoodieStorage storage) {
this.writeConfig = writeConfig;
- this.storageConf = storageConf;
+ this.storage = storage;
}
public abstract Timer.Context getOverallTimerContext();
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java
index 24d4b1260f2..6094edeeedb 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java
@@ -34,6 +34,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.timeline.service.TimelineService;
import org.apache.hudi.utilities.UtilHelpers;
@@ -100,7 +101,9 @@ public class TimelineServerPerf implements Serializable {
public void run() throws IOException {
JavaSparkContext jsc = UtilHelpers.buildSparkContext("hudi-view-perf-" + cfg.basePath, cfg.sparkMaster);
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
- List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(engineContext, cfg.basePath, cfg.useFileListingFromMetadata);
+ List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(
+ engineContext, new HoodieHadoopStorage(cfg.basePath, engineContext.getStorageConf()),
+ cfg.basePath, cfg.useFileListingFromMetadata);
Collections.shuffle(allPartitionPaths);
List<String> selected = allPartitionPaths.stream().filter(p -> !p.contains("error")).limit(cfg.maxPartitions)
.collect(Collectors.toList());
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java
index c5c01bee231..41462dc4198 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java
@@ -22,7 +22,7 @@ package org.apache.hudi.utilities.streamer;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.metrics.Metrics;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.utilities.ingestion.HoodieIngestionMetrics;
import com.codahale.metrics.Timer;
@@ -38,14 +38,14 @@ public class HoodieStreamerMetrics extends HoodieIngestionMetrics {
private transient Timer hiveSyncTimer;
private transient Timer metaSyncTimer;
- public HoodieStreamerMetrics(HoodieWriteConfig writeConfig, StorageConfiguration<?> storageConf) {
- this(writeConfig.getMetricsConfig(), storageConf);
+ public HoodieStreamerMetrics(HoodieWriteConfig writeConfig, HoodieStorage storage) {
+ this(writeConfig.getMetricsConfig(), storage);
}
- public HoodieStreamerMetrics(HoodieMetricsConfig writeConfig, StorageConfiguration<?> storageConf) {
- super(writeConfig, storageConf);
+ public HoodieStreamerMetrics(HoodieMetricsConfig writeConfig, HoodieStorage storage) {
+ super(writeConfig, storage);
if (writeConfig.isMetricsOn()) {
- metrics = Metrics.getInstance(writeConfig, storageConf);
+ metrics = Metrics.getInstance(writeConfig, storage);
this.overallTimerName = getMetricsName("timer", "deltastreamer");
this.hiveSyncTimerName = getMetricsName("timer", "deltastreamerHiveSync");
this.metaSyncTimerName = getMetricsName("timer", "deltastreamerMetaSync");
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
index 365e128cc8c..9778301f2f6 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
@@ -75,7 +75,6 @@ import org.apache.hudi.keygen.KeyGenUtils;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.sync.common.util.SyncUtilHelpers;
@@ -314,9 +313,9 @@ public class StreamSync implements Serializable, Closeable {
HoodieWriteConfig hoodieWriteConfig = getHoodieClientConfig();
this.metrics = (HoodieIngestionMetrics) ReflectionUtils.loadClass(cfg.ingestionMetricsClass,
- new Class<?>[] { HoodieMetricsConfig.class, StorageConfiguration.class},
- hoodieWriteConfig.getMetricsConfig(), storage.getConf());
- this.hoodieMetrics = new HoodieMetrics(hoodieWriteConfig, storage.getConf());
+ new Class<?>[] {HoodieMetricsConfig.class, HoodieStorage.class},
+ hoodieWriteConfig.getMetricsConfig(), storage);
+ this.hoodieMetrics = new HoodieMetrics(hoodieWriteConfig, storage);
if (props.getBoolean(ERROR_TABLE_ENABLED.key(), ERROR_TABLE_ENABLED.defaultValue())) {
this.errorTableWriter = ErrorTableUtils.getErrorTableWriter(
cfg, sparkSession, props, hoodieSparkContext, fs);
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
index 9b06c333aa1..3f26ed7b5b8 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
@@ -205,7 +205,7 @@ public class TestHoodieIndexer extends SparkClientFunctionalTestHarness implemen
metaClient.getActiveTimeline().readIndexPlanAsBytes(indexingInstant).get());
String indexUptoInstantTime = indexPlan.getIndexPartitionInfos().get(0).getIndexUptoInstant();
HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(
- context(), metadataConfig, metaClient.getBasePathV2().toString());
+ context(), metaClient.getStorage(), metadataConfig, metaClient.getBasePathV2().toString());
HoodieTableMetaClient metadataMetaClient = metadata.getMetadataMetaClient();
String mdtCommitTime = indexingInstant.getTimestamp();
assertTrue(metadataMetaClient.getActiveTimeline().containsInstant(mdtCommitTime));
@@ -270,7 +270,7 @@ public class TestHoodieIndexer extends SparkClientFunctionalTestHarness implemen
metaClient.getActiveTimeline().revertToInflight(commit);
HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(
- context(), metadataConfig, metaClient.getBasePathV2().toString());
+ context(), metaClient.getStorage(), metadataConfig, metaClient.getBasePathV2().toString());
HoodieTableMetaClient metadataMetaClient = metadata.getMetadataMetaClient();
HoodieInstant mdtCommit = metadataMetaClient.getActiveTimeline()
.filter(i -> i.getTimestamp().equals(commitTime))
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
index 136c82a9d98..895919da016 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
@@ -20,8 +20,8 @@ package org.apache.hudi.utilities;
import org.apache.hudi.DataSourceWriteOptions;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
-import org.apache.hudi.common.config.HoodieTimeGeneratorConfig;
import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.config.HoodieTimeGeneratorConfig;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -205,7 +205,7 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
}
@Override
- List<String> getPartitionsFromMDT(HoodieEngineContext engineContext, String basePath) {
+ List<String> getPartitionsFromMDT(HoodieEngineContext engineContext, String basePath, HoodieStorage storage) {
return metadataPartitionsToReturn;
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index 84dd5986114..162eb2c35d3 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -2883,7 +2883,8 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
syncAndAssertRecordCount(cfg, 1000, tableBasePath, "00000", 1);
HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setBasePath(tableBasePath).setConf(HoodieTestUtils.getDefaultStorageConf()).build();
- List<String> partitions = FSUtils.getAllPartitionPaths(new HoodieLocalEngineContext(metaClient.getStorageConf()), metaClient.getBasePath(), false);
+ List<String> partitions = FSUtils.getAllPartitionPaths(
+ new HoodieLocalEngineContext(metaClient.getStorageConf()), metaClient.getStorage(), metaClient.getBasePath(), false);
StorageConfiguration hadoopConf = metaClient.getStorageConf();
HoodieLocalEngineContext engContext = new HoodieLocalEngineContext(hadoopConf);
HoodieMetadataFileSystemView fsView = new HoodieMetadataFileSystemView(engContext, metaClient,
@@ -2894,7 +2895,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
assertEquals(baseFiles.size(), partitions.size());
// Verify if each parquet file is actually sorted by sortColumn.
for (String filePath : baseFiles) {
- try (HoodieAvroParquetReader parquetReader = new HoodieAvroParquetReader(HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath))) {
+ try (HoodieAvroParquetReader parquetReader = new HoodieAvroParquetReader(HoodieTestUtils.getStorage(filePath), new StoragePath(filePath))) {
ClosableIterator<HoodieRecord<IndexedRecord>> iterator = parquetReader.getRecordIterator();
List<Float> sortColumnValues = new ArrayList<>();
while (iterator.hasNext()) {