You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/05/25 09:04:08 UTC
(hudi) branch release-0.15.0 updated (0de2e8018ff -> b6078994449)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch release-0.15.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 0de2e8018ff [HUDI-7786] Fix roaring bitmap dependency in hudi-integ-test-bundle (#11283)
new b71da938ac5 [HUDI-7785] Keep public APIs in utilities module the same as before HoodieStorage abstraction (#11280)
new 670f13102f0 [HUDI-7775] Remove unused APIs in HoodieStorage (#11281)
new b350a26cabf [HUDI-7788] Fixing exception handling in AverageRecordSizeUtils (#11290)
new 77b1ded0209 [HUDI-7776] Simplify HoodieStorage instance fetching (#11292)
new 2e763c9ec8e [HUDI-7792] Bump h2 from 1.4.200 to 2.2.220 (#11296)
new 72dd5183ba5 [HUDI-7790] Revert changes in DFSPathSelector and UtilHelpers.readConfig (#11294)
new b6078994449 [HUDI-7794] Bump org.apache.hive:hive-service from 2.3.1 to 2.3.4 (#11298)
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hudi/cli/commands/ArchivedCommitsCommand.java | 2 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 22 ++---
.../apache/hudi/cli/commands/RepairsCommand.java | 2 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 3 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 2 +-
.../apache/hudi/client/HoodieTimelineArchiver.java | 2 +-
.../hudi/client/heartbeat/HeartbeatUtils.java | 5 +-
.../hudi/client/transaction/lock/LockManager.java | 3 +-
.../lock/metrics/HoodieLockMetrics.java | 14 +--
.../org/apache/hudi/index/HoodieIndexUtils.java | 7 +-
.../hudi/index/bloom/HoodieGlobalBloomIndex.java | 2 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 6 +-
.../hudi/index/simple/HoodieGlobalSimpleIndex.java | 2 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 2 +-
.../hudi/io/HoodieKeyLocationFetchHandle.java | 4 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieReadHandle.java | 8 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 8 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 6 +-
.../java/org/apache/hudi/table/HoodieTable.java | 10 +-
.../table/action/clean/CleanActionExecutor.java | 6 +-
.../hudi/table/action/clean/CleanPlanner.java | 4 +-
.../PartitionAwareClusteringPlanStrategy.java | 3 +-
.../action/commit/BaseCommitActionExecutor.java | 2 +-
.../table/action/commit/HoodieMergeHelper.java | 13 +--
.../compact/RunCompactionActionExecutor.java | 2 +-
.../BaseHoodieCompactionPlanGenerator.java | 2 +-
.../index/RecordBasedIndexingCatchupTask.java | 3 +-
.../table/action/index/RunIndexActionExecutor.java | 6 +-
.../action/index/ScheduleIndexActionExecutor.java | 2 +-
.../action/restore/BaseRestoreActionExecutor.java | 2 +-
.../rollback/BaseRollbackActionExecutor.java | 4 +-
.../rollback/ListingBasedRollbackStrategy.java | 2 +-
.../action/savepoint/SavepointActionExecutor.java | 3 +-
.../hudi/table/marker/DirectWriteMarkers.java | 2 +-
.../table/marker/MarkerBasedRollbackUtils.java | 2 +-
.../hudi/avro/TestHoodieAvroParquetWriter.java | 16 ++--
.../hudi/client/utils/TestCommitMetadataUtils.java | 2 +-
.../hudi/metrics/TestHoodieConsoleMetrics.java | 4 +-
.../hudi/metrics/TestHoodieGraphiteMetrics.java | 4 +-
.../apache/hudi/metrics/TestHoodieJmxMetrics.java | 4 +-
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 4 +-
.../datadog/TestDatadogMetricsReporter.java | 10 +-
.../org/apache/hudi/metrics/m3/TestM3Metrics.java | 21 ++---
.../metrics/prometheus/TestPrometheusReporter.java | 6 +-
.../prometheus/TestPushGateWayReporter.java | 6 +-
.../GenericRecordValidationTestUtils.java | 7 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 2 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 7 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 2 +-
.../FlinkDeletePartitionCommitActionExecutor.java | 2 +-
.../index/bloom/TestFlinkHoodieBloomIndex.java | 3 +-
.../run/strategy/JavaExecutionStrategy.java | 7 +-
.../JavaHoodieBackedTableMetadataWriter.java | 2 +-
...vaInsertOverwriteTableCommitActionExecutor.java | 2 +-
.../client/TestHoodieJavaWriteClientInsert.java | 8 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 10 +-
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 2 +-
.../commit/TestJavaCopyOnWriteActionExecutor.java | 14 +--
.../testutils/HoodieJavaClientTestHarness.java | 11 ++-
.../MultipleSparkJobExecutionStrategy.java | 14 ++-
.../strategy/SingleSparkJobExecutionStrategy.java | 3 +-
.../client/validator/SparkPreCommitValidator.java | 2 +-
.../index/bloom/HoodieFileProbingFunction.java | 3 +-
.../io/storage/HoodieSparkFileReaderFactory.java | 18 ++--
.../io/storage/HoodieSparkFileWriterFactory.java | 11 ++-
.../hudi/io/storage/HoodieSparkIOFactory.java | 15 +--
.../hudi/io/storage/HoodieSparkParquetReader.java | 32 +++----
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 4 +-
.../SparkHoodieBackedTableMetadataWriter.java | 2 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 2 +-
.../action/commit/AverageRecordSizeUtils.java | 5 +-
.../SparkDeletePartitionCommitActionExecutor.java | 2 +-
...rkInsertOverwriteTableCommitActionExecutor.java | 3 +-
.../org/apache/hudi/client/TestClientRollback.java | 4 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 2 +-
.../functional/TestExternalPathHandling.java | 3 +-
.../functional/TestHoodieBackedMetadata.java | 40 ++++----
.../functional/TestHoodieBackedTableMetadata.java | 11 ++-
.../TestHoodieClientOnCopyOnWriteStorage.java | 30 +++---
.../TestRemoteFileSystemViewWithMetadataTable.java | 2 +-
.../hudi/index/bloom/TestHoodieBloomIndex.java | 2 +-
.../storage/TestHoodieAvroFileWriterFactory.java | 8 +-
.../action/commit/TestAverageRecordSizeUtils.java | 20 ++++
.../commit/TestCopyOnWriteActionExecutor.java | 6 +-
.../table/functional/TestHoodieSparkRollback.java | 7 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 3 +-
.../TestTimelineServerBasedWriteMarkers.java | 3 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 6 +-
.../hudi/testutils/FunctionalTestHarness.java | 3 +-
.../testutils/HoodieSparkClientTestHarness.java | 4 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 7 +-
.../hudi/common/config/HoodieStorageConfig.java | 12 ++-
.../java/org/apache/hudi/common/fs/FSUtils.java | 12 ++-
.../hudi/common/model/HoodieCommitMetadata.java | 16 ++--
.../hudi/common/model/HoodiePartitionMetadata.java | 2 +-
.../hudi/common/table/HoodieTableMetaClient.java | 7 +-
.../hudi/common/table/TableSchemaResolver.java | 4 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 7 +-
.../table/log/block/HoodieAvroDataBlock.java | 4 +-
.../common/table/log/block/HoodieCommandBlock.java | 4 +-
.../common/table/log/block/HoodieCorruptBlock.java | 4 +-
.../common/table/log/block/HoodieDataBlock.java | 8 +-
.../common/table/log/block/HoodieDeleteBlock.java | 4 +-
.../table/log/block/HoodieHFileDataBlock.java | 20 ++--
.../common/table/log/block/HoodieLogBlock.java | 14 +--
.../table/log/block/HoodieParquetDataBlock.java | 12 ++-
.../table/timeline/HoodieActiveTimeline.java | 8 +-
.../common/table/view/FileSystemViewManager.java | 2 +-
.../apache/hudi/common/util/FileFormatUtils.java | 101 ++++++++++----------
.../hudi/common/util/InternalSchemaCache.java | 16 ++--
.../io/FileBasedInternalSchemaStorageManager.java | 15 +--
.../hudi/io/storage/HoodieFileReaderFactory.java | 7 +-
.../hudi/io/storage/HoodieFileWriterFactory.java | 16 ++--
.../apache/hudi/io/storage/HoodieIOFactory.java | 13 ++-
.../io/storage/HoodieNativeAvroHFileReader.java | 13 +--
.../hudi/metadata/AbstractHoodieTableMetadata.java | 7 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 25 +++--
.../metadata/FileSystemBackedTableMetadata.java | 29 +++---
.../hudi/metadata/HoodieBackedTableMetadata.java | 17 +++-
.../metadata/HoodieMetadataFileSystemView.java | 5 +-
.../hudi/metadata/HoodieMetadataMetrics.java | 6 +-
.../hudi/metadata/HoodieMetadataPayload.java | 11 ---
.../apache/hudi/metadata/HoodieTableMetadata.java | 24 +++--
.../hudi/metadata/HoodieTableMetadataUtil.java | 23 ++---
.../main/java/org/apache/hudi/metrics/Metrics.java | 16 ++--
.../apache/hudi/storage/HoodieStorageUtils.java | 13 ++-
.../hudi/common/testutils/HoodieTestUtils.java | 6 ++
.../HoodieBackedTestDelayedTableMetadata.java | 4 +-
.../hudi/sink/bootstrap/BootstrapOperator.java | 6 +-
.../hudi/sink/clustering/ClusteringOperator.java | 6 +-
.../sink/partitioner/profile/WriteProfiles.java | 5 +-
.../java/org/apache/hudi/source/FileIndex.java | 9 +-
.../hudi/source/stats/ColumnStatsIndices.java | 4 +-
.../hudi/table/format/InternalSchemaManager.java | 6 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 5 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 2 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 2 +-
.../common/config/DFSPropertiesConfiguration.java | 3 +-
.../common/table/log/HoodieLogFormatWriter.java | 2 +-
.../org/apache/hudi/common/util/HFileUtils.java | 29 +++---
.../java/org/apache/hudi/common/util/OrcUtils.java | 62 ++++++-------
.../org/apache/hudi/common/util/ParquetUtils.java | 103 ++++++++++-----------
.../io/hadoop/HoodieAvroFileReaderFactory.java | 19 ++--
.../io/hadoop/HoodieAvroFileWriterFactory.java | 16 ++--
.../apache/hudi/io/hadoop/HoodieAvroOrcReader.java | 22 ++---
.../hudi/io/hadoop/HoodieAvroParquetReader.java | 19 ++--
.../{storage => hadoop}/HoodieHadoopIOFactory.java | 24 ++---
.../hudi/storage/hadoop/HoodieHadoopStorage.java | 41 ++++----
.../common/functional/TestHoodieLogFormat.java | 4 +-
.../hudi/common/table/TestTableSchemaResolver.java | 7 +-
.../table/log/block/TestHoodieDeleteBlock.java | 2 +-
.../util/TestDFSPropertiesConfiguration.java | 6 +-
.../apache/hudi/common/util/TestParquetUtils.java | 14 +--
.../io/hadoop/TestHoodieAvroFileReaderFactory.java | 13 ++-
.../hadoop/TestHoodieHBaseHFileReaderWriter.java | 17 ++--
.../io/hadoop/TestHoodieHFileReaderWriter.java | 18 ++--
.../io/hadoop/TestHoodieHFileReaderWriterBase.java | 42 ++++-----
.../hudi/io/hadoop/TestHoodieOrcReaderWriter.java | 13 +--
.../hudi/io/hadoop/TestHoodieReaderWriterBase.java | 43 +++++----
.../TestFileSystemBackedTableMetadata.java | 12 +--
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 2 +-
.../hudi/hadoop/HoodieHFileRecordReader.java | 3 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 5 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 5 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 4 +-
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 3 +-
.../SparkDataSourceContinuousIngestTool.java | 3 +-
.../helpers/DFSTestSuitePathSelector.java | 41 ++++----
.../reader/DFSHoodieDatasetInputReader.java | 4 +-
.../org/apache/hudi/storage/HoodieStorage.java | 52 +++++------
.../hudi/io/storage/TestHoodieStorageBase.java | 9 --
hudi-platform-service/hudi-metaserver/pom.xml | 2 +-
...lkInsertOverwriteTableCommitActionExecutor.java | 4 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 2 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 7 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 18 ++--
.../src/main/scala/org/apache/hudi/Iterators.scala | 2 +-
.../org/apache/hudi/RecordLevelIndexSupport.scala | 4 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 2 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 20 ++--
.../hudi/command/RepairHoodieTableCommand.scala | 16 +++-
.../sql/hudi/streaming/HoodieStreamSource.scala | 3 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 14 +--
.../RepairMigratePartitionMetaProcedure.scala | 3 +-
.../procedures/ShowInvalidParquetProcedure.scala | 6 +-
.../ShowMetadataTableFilesProcedure.scala | 4 +-
.../ShowMetadataTablePartitionsProcedure.scala | 7 +-
.../ShowMetadataTableStatsProcedure.scala | 4 +-
.../ValidateMetadataTableFilesProcedure.scala | 5 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 18 ++--
.../org/apache/hudi/functional/TestBootstrap.java | 14 +--
.../TestSparkConsistentBucketClustering.java | 4 +-
.../row/TestHoodieInternalRowParquetWriter.java | 4 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 1 +
.../apache/hudi/functional/TestCOWDataSource.scala | 8 +-
.../hudi/functional/TestColumnStatsIndex.scala | 8 +-
.../TestMetadataTableWithSparkDataSource.scala | 13 ++-
.../hudi/dml/TestPartialUpdateForMergeInto.scala | 5 +
.../Spark30LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark31LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark32LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark33LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark34LegacyHoodieParquetFileFormat.scala | 5 +-
.../Spark35LegacyHoodieParquetFileFormat.scala | 4 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 1 +
.../hudi/sync/common/util/ManifestFileWriter.java | 2 +-
.../apache/hudi/utilities/HDFSParquetImporter.java | 3 +-
.../org/apache/hudi/utilities/HoodieCleaner.java | 3 +-
.../apache/hudi/utilities/HoodieClusteringJob.java | 4 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 4 +-
.../hudi/utilities/HoodieDataTableValidator.java | 5 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 4 +-
.../org/apache/hudi/utilities/HoodieIndexer.java | 4 +-
.../utilities/HoodieMetadataTableValidator.java | 18 ++--
.../apache/hudi/utilities/HoodieRepairTool.java | 4 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 3 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 7 +-
.../org/apache/hudi/utilities/TableSizeStats.java | 7 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 8 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 3 +-
.../deltastreamer/HoodieDeltaStreamer.java | 5 +-
.../deltastreamer/HoodieDeltaStreamerMetrics.java | 10 +-
.../ingestion/HoodieIngestionMetrics.java | 12 +--
.../hudi/utilities/perf/TimelineServerPerf.java | 5 +-
.../utilities/sources/helpers/DFSPathSelector.java | 54 +++++------
.../sources/helpers/DatePartitionPathSelector.java | 46 ++++-----
.../utilities/streamer/BaseErrorTableWriter.java | 13 ++-
.../hudi/utilities/streamer/ErrorTableUtils.java | 5 +-
.../streamer/HoodieMultiTableStreamer.java | 5 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 46 +++++----
.../utilities/streamer/HoodieStreamerMetrics.java | 12 +--
.../apache/hudi/utilities/streamer/StreamSync.java | 15 ++-
.../apache/hudi/utilities/TestHoodieIndexer.java | 4 +-
.../TestHoodieMetadataTableValidator.java | 2 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 4 +-
...TestHoodieDeltaStreamerSchemaEvolutionBase.java | 6 +-
.../utilities/sources/TestJsonKafkaSource.java | 2 +-
.../helpers/TestDFSPathSelectorCommonMethods.java | 19 ++--
.../helpers/TestDatePartitionPathSelector.java | 5 +-
.../utilities/streamer/TestErrorTableUtils.java | 70 ++++++++++++++
packaging/hudi-flink-bundle/pom.xml | 2 +-
packaging/hudi-metaserver-server-bundle/pom.xml | 2 +-
pom.xml | 2 +-
246 files changed, 1309 insertions(+), 1163 deletions(-)
rename hudi-hadoop-common/src/main/java/org/apache/hudi/io/{storage => hadoop}/HoodieHadoopIOFactory.java (81%)
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestErrorTableUtils.java
(hudi) 06/07: [HUDI-7790] Revert changes in DFSPathSelector and UtilHelpers.readConfig (#11294)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch release-0.15.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 72dd5183ba54e2885792d481f30db35166bef218
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Sat May 25 00:18:00 2024 -0700
[HUDI-7790] Revert changes in DFSPathSelector and UtilHelpers.readConfig (#11294)
---
.../org/apache/hudi/cli/commands/SparkMain.java | 3 +-
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 3 +-
.../SparkDataSourceContinuousIngestTool.java | 3 +-
.../helpers/DFSTestSuitePathSelector.java | 41 ++++++++--------
.../apache/hudi/utilities/HDFSParquetImporter.java | 3 +-
.../org/apache/hudi/utilities/HoodieCleaner.java | 3 +-
.../apache/hudi/utilities/HoodieClusteringJob.java | 4 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 4 +-
.../hudi/utilities/HoodieDataTableValidator.java | 3 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 4 +-
.../org/apache/hudi/utilities/HoodieIndexer.java | 4 +-
.../utilities/HoodieMetadataTableValidator.java | 2 +-
.../apache/hudi/utilities/HoodieRepairTool.java | 2 +-
.../org/apache/hudi/utilities/TableSizeStats.java | 3 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 8 ++--
.../utilities/sources/helpers/DFSPathSelector.java | 54 ++++++++++------------
.../sources/helpers/DatePartitionPathSelector.java | 46 +++++++-----------
.../streamer/HoodieMultiTableStreamer.java | 5 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 5 +-
.../helpers/TestDFSPathSelectorCommonMethods.java | 19 ++++----
.../helpers/TestDatePartitionPathSelector.java | 5 +-
21 files changed, 105 insertions(+), 119 deletions(-)
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
index f8106ffc55c..fe13813490d 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
@@ -44,7 +44,6 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.keygen.constant.KeyGeneratorType;
import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.action.compact.strategy.UnBoundedCompactionStrategy;
import org.apache.hudi.table.marker.WriteMarkersFactory;
@@ -483,7 +482,7 @@ public class SparkMain {
String payloadClassName, String enableHiveSync, String propsFilePath, List<String> configs) throws IOException {
TypedProperties properties = propsFilePath == null ? buildProperties(configs)
- : readConfig(jsc.hadoopConfiguration(), new StoragePath(propsFilePath), configs).getProps(true);
+ : readConfig(jsc.hadoopConfiguration(), new Path(propsFilePath), configs).getProps(true);
properties.setProperty(HoodieBootstrapConfig.BASE_PATH.key(), sourcePath);
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java
index 8813129d748..70910357d7d 100644
--- a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java
@@ -44,7 +44,6 @@ import org.apache.hudi.integ.testsuite.reader.DeltaInputType;
import org.apache.hudi.integ.testsuite.writer.DeltaOutputMode;
import org.apache.hudi.keygen.BuiltinKeyGenerator;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.utilities.UtilHelpers;
import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer;
@@ -115,7 +114,7 @@ public class HoodieTestSuiteJob {
SparkSession.builder().config(jsc.getConf()).enableHiveSupport().getOrCreate();
this.fs = HadoopFSUtils.getFs(cfg.inputBasePath, jsc.hadoopConfiguration());
this.props =
- UtilHelpers.readConfig(fs.getConf(), new StoragePath(cfg.propsFilePath), cfg.configs).getProps();
+ UtilHelpers.readConfig(fs.getConf(), new Path(cfg.propsFilePath), cfg.configs).getProps();
log.info("Creating workload generator with configs : {}", props.toString());
this.hiveConf = getDefaultHiveConf(jsc.hadoopConfiguration());
this.keyGenerator =
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
index cbb2a27e54f..81bc4435623 100644
--- a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
@@ -22,7 +22,6 @@ package org.apache.hudi.integ.testsuite;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.utilities.HoodieRepairTool;
import org.apache.hudi.utilities.IdentitySplitter;
import org.apache.hudi.utilities.UtilHelpers;
@@ -133,7 +132,7 @@ public class SparkDataSourceContinuousIngestTool {
* @return the {@link TypedProperties} instance.
*/
private TypedProperties readConfigFromFileSystem(JavaSparkContext jsc, Config cfg) {
- return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new StoragePath(cfg.propsFilePath), cfg.configs)
+ return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new Path(cfg.propsFilePath), cfg.configs)
.getProps(true);
}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/DFSTestSuitePathSelector.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/DFSTestSuitePathSelector.java
index e2a2c19f666..70026aa5f7f 100644
--- a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/DFSTestSuitePathSelector.java
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/DFSTestSuitePathSelector.java
@@ -24,17 +24,20 @@ import org.apache.hudi.common.util.collection.ImmutablePair;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.integ.testsuite.HoodieTestSuiteJob;
-import org.apache.hudi.storage.StoragePathInfo;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.utilities.config.DFSPathSelectorConfig;
import org.apache.hudi.utilities.sources.helpers.DFSPathSelector;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
@@ -67,31 +70,31 @@ public class DFSTestSuitePathSelector extends DFSPathSelector {
}
// obtain all eligible files for the batch
- List<StoragePathInfo> eligibleFiles = new ArrayList<>();
- List<StoragePathInfo> pathInfoList = storage.globEntries(
- new StoragePath(getStringWithAltKeys(props, DFSPathSelectorConfig.ROOT_INPUT_PATH),
- "*"));
+ List<FileStatus> eligibleFiles = new ArrayList<>();
+ FileStatus[] fileStatuses = fs.globStatus(
+ new Path(getStringWithAltKeys(props, DFSPathSelectorConfig.ROOT_INPUT_PATH), "*"));
// Say input data is as follow input/1, input/2, input/5 since 3,4 was rolled back and 5 is new generated data
// checkpoint from the latest commit metadata will be 2 since 3,4 has been rolled back. We need to set the
// next batch id correctly as 5 instead of 3
- Option<String> correctBatchIdDueToRollback = Option.fromJavaOptional(pathInfoList.stream()
- .map(f -> f.getPath().toString().split("/")[
- f.getPath().toString().split("/").length - 1])
+ Option<String> correctBatchIdDueToRollback = Option.fromJavaOptional(Arrays.stream(fileStatuses)
+ .map(f -> f.getPath().toString().split("/")[f.getPath().toString().split("/").length - 1])
.filter(bid1 -> Integer.parseInt(bid1) > lastBatchId)
.min((bid1, bid2) -> Integer.min(Integer.parseInt(bid1), Integer.parseInt(bid2))));
- if (correctBatchIdDueToRollback.isPresent()
- && Integer.parseInt(correctBatchIdDueToRollback.get()) > nextBatchId) {
+ if (correctBatchIdDueToRollback.isPresent() && Integer.parseInt(correctBatchIdDueToRollback.get()) > nextBatchId) {
nextBatchId = Integer.parseInt(correctBatchIdDueToRollback.get());
}
- log.info("Using DFSTestSuitePathSelector, checkpoint: " + lastCheckpointStr + " sourceLimit: "
- + sourceLimit + " lastBatchId: " + lastBatchId + " nextBatchId: " + nextBatchId);
- for (StoragePathInfo pathInfo : pathInfoList) {
- if (!pathInfo.isDirectory() || IGNORE_FILEPREFIX_LIST.stream()
- .anyMatch(pfx -> pathInfo.getPath().getName().startsWith(pfx))) {
+ log.info("Using DFSTestSuitePathSelector, checkpoint: " + lastCheckpointStr + " sourceLimit: " + sourceLimit
+ + " lastBatchId: " + lastBatchId + " nextBatchId: " + nextBatchId);
+ for (FileStatus fileStatus : fileStatuses) {
+ if (!fileStatus.isDirectory() || IGNORE_FILEPREFIX_LIST.stream()
+ .anyMatch(pfx -> fileStatus.getPath().getName().startsWith(pfx))) {
continue;
- } else if (Integer.parseInt(pathInfo.getPath().getName()) > lastBatchId
- && Integer.parseInt(pathInfo.getPath().getName()) <= nextBatchId) {
- eligibleFiles.addAll(storage.listFiles(pathInfo.getPath()));
+ } else if (Integer.parseInt(fileStatus.getPath().getName()) > lastBatchId && Integer.parseInt(fileStatus.getPath()
+ .getName()) <= nextBatchId) {
+ RemoteIterator<LocatedFileStatus> files = fs.listFiles(fileStatus.getPath(), true);
+ while (files.hasNext()) {
+ eligibleFiles.add(files.next());
+ }
}
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java
index 3513f7c6760..1dc24fd31b8 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java
@@ -32,7 +32,6 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.utilities.streamer.HoodieStreamer;
import com.beust.jcommander.IValueValidator;
@@ -114,7 +113,7 @@ public class HDFSParquetImporter implements Serializable {
public int dataImport(JavaSparkContext jsc, int retry) {
this.fs = HadoopFSUtils.getFs(cfg.targetPath, jsc.hadoopConfiguration());
this.props = cfg.propsFilePath == null ? UtilHelpers.buildProperties(cfg.configs)
- : UtilHelpers.readConfig(fs.getConf(), new StoragePath(cfg.propsFilePath), cfg.configs).getProps(true);
+ : UtilHelpers.readConfig(fs.getConf(), new Path(cfg.propsFilePath), cfg.configs).getProps(true);
LOG.info("Starting data import with configs : " + props.toString());
int ret = -1;
try {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java
index e1d6a13cb9a..83f535191b9 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java
@@ -23,7 +23,6 @@ import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.storage.StoragePath;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
@@ -62,7 +61,7 @@ public class HoodieCleaner {
* Filesystem used.
*/
this.props = cfg.propsFilePath == null ? UtilHelpers.buildProperties(cfg.configs)
- : UtilHelpers.readConfig(jssc.hadoopConfiguration(), new StoragePath(cfg.propsFilePath), cfg.configs).getProps(true);
+ : UtilHelpers.readConfig(jssc.hadoopConfiguration(), new Path(cfg.propsFilePath), cfg.configs).getProps(true);
LOG.info("Creating Cleaner with configs : " + props.toString());
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java
index b96b4610376..90c7d493705 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java
@@ -29,11 +29,11 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.config.HoodieCleanConfig;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieSparkTable;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
+import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaSparkContext;
import org.jetbrains.annotations.TestOnly;
import org.slf4j.Logger;
@@ -73,7 +73,7 @@ public class HoodieClusteringJob {
}
private TypedProperties readConfigFromFileSystem(JavaSparkContext jsc, Config cfg) {
- return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new StoragePath(cfg.propsFilePath), cfg.configs)
+ return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new Path(cfg.propsFilePath), cfg.configs)
.getProps(true);
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java
index 90c66add046..82acce6a4eb 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java
@@ -30,7 +30,6 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.config.HoodieCleanConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.table.action.compact.strategy.LogFileSizeBasedCompactionStrategy;
@@ -38,6 +37,7 @@ import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import org.apache.avro.Schema;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.slf4j.Logger;
@@ -76,7 +76,7 @@ public class HoodieCompactor {
}
private TypedProperties readConfigFromFileSystem(JavaSparkContext jsc, Config cfg) {
- return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new StoragePath(cfg.propsFilePath), cfg.configs)
+ return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new Path(cfg.propsFilePath), cfg.configs)
.getProps(true);
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
index 459483e547c..9953b5225a3 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
@@ -39,6 +39,7 @@ import org.apache.hudi.table.repair.RepairUtils;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
+import org.apache.hadoop.fs.Path;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.slf4j.Logger;
@@ -139,7 +140,7 @@ public class HoodieDataTableValidator implements Serializable {
* @return the {@link TypedProperties} instance.
*/
private TypedProperties readConfigFromFileSystem(JavaSparkContext jsc, Config cfg) {
- return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new StoragePath(cfg.propsFilePath), cfg.configs)
+ return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new Path(cfg.propsFilePath), cfg.configs)
.getProps(true);
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
index 17210d25639..05a5742e841 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
@@ -34,7 +34,6 @@ import org.apache.hudi.hive.HiveSyncConfig;
import org.apache.hudi.hive.HiveSyncConfigHolder;
import org.apache.hudi.hive.HiveSyncTool;
import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.sync.common.HoodieSyncConfig;
import org.apache.hudi.table.HoodieSparkTable;
@@ -42,6 +41,7 @@ import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
@@ -136,7 +136,7 @@ public class HoodieDropPartitionsTool implements Serializable {
* @return the {@link TypedProperties} instance.
*/
private TypedProperties readConfigFromFileSystem(JavaSparkContext jsc, Config cfg) {
- return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new StoragePath(cfg.propsFilePath), cfg.configs)
+ return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new Path(cfg.propsFilePath), cfg.configs)
.getProps(true);
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java
index 13d168a24c0..5c626a53ae7 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java
@@ -31,10 +31,10 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.metadata.MetadataPartitionType;
-import org.apache.hudi.storage.StoragePath;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
+import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaSparkContext;
import org.jetbrains.annotations.TestOnly;
import org.slf4j.Logger;
@@ -105,7 +105,7 @@ public class HoodieIndexer {
}
private TypedProperties readConfigFromFileSystem(JavaSparkContext jsc, HoodieIndexer.Config cfg) {
- return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new StoragePath(cfg.propsFilePath), cfg.configs)
+ return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new Path(cfg.propsFilePath), cfg.configs)
.getProps(true);
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
index a9bade03137..bfb9e18af1b 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
@@ -258,7 +258,7 @@ public class HoodieMetadataTableValidator implements Serializable {
* @return the {@link TypedProperties} instance.
*/
private TypedProperties readConfigFromFileSystem(JavaSparkContext jsc, Config cfg) {
- return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new StoragePath(cfg.propsFilePath), cfg.configs)
+ return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new Path(cfg.propsFilePath), cfg.configs)
.getProps(true);
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java
index b2bb34ede3b..237e0cb2263 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java
@@ -518,7 +518,7 @@ public class HoodieRepairTool {
* @return the {@link TypedProperties} instance.
*/
private TypedProperties readConfigFromFileSystem(JavaSparkContext jsc, Config cfg) {
- return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new StoragePath(cfg.propsFilePath), cfg.configs)
+ return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new Path(cfg.propsFilePath), cfg.configs)
.getProps(true);
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
index ff655dfd017..a9b0f70bca9 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
@@ -34,7 +34,6 @@ import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.storage.StorageConfiguration;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import com.beust.jcommander.JCommander;
@@ -132,7 +131,7 @@ public class TableSizeStats implements Serializable {
* @return the {@link TypedProperties} instance.
*/
private TypedProperties readConfigFromFileSystem(JavaSparkContext jsc, Config cfg) {
- return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new StoragePath(cfg.propsFilePath), cfg.configs)
+ return UtilHelpers.readConfig(jsc.hadoopConfiguration(), new Path(cfg.propsFilePath), cfg.configs)
.getProps(true);
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
index abf0558e5ff..74cc775718a 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
@@ -118,6 +118,7 @@ import java.util.function.Supplier;
import static org.apache.hudi.common.util.ConfigUtils.getBooleanWithAltKeys;
import static org.apache.hudi.common.util.ConfigUtils.getStringWithAltKeys;
+import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath;
/**
* Bunch of helper methods.
@@ -242,13 +243,14 @@ public class UtilHelpers {
}
public static DFSPropertiesConfiguration readConfig(Configuration hadoopConfig,
- StoragePath cfgPath,
+ Path cfgPath,
List<String> overriddenProps) {
- DFSPropertiesConfiguration conf = new DFSPropertiesConfiguration(hadoopConfig, cfgPath);
+ StoragePath storagePath = convertToStoragePath(cfgPath);
+ DFSPropertiesConfiguration conf = new DFSPropertiesConfiguration(hadoopConfig, storagePath);
try {
if (!overriddenProps.isEmpty()) {
LOG.info("Adding overridden properties to file properties.");
- conf.addPropsFromStream(new BufferedReader(new StringReader(String.join("\n", overriddenProps))), cfgPath);
+ conf.addPropsFromStream(new BufferedReader(new StringReader(String.join("\n", overriddenProps))), storagePath);
}
} catch (IOException ioe) {
throw new HoodieIOException("Unexpected error adding config overrides", ioe);
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java
index 62f182df359..257c015c53b 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java
@@ -26,13 +26,12 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
-import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.utilities.config.DFSPathSelectorConfig;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaSparkContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,15 +65,15 @@ public class DFSPathSelector implements Serializable {
protected static final List<String> IGNORE_FILEPREFIX_LIST = Arrays.asList(".", "_");
- protected final transient HoodieStorage storage;
+ protected final transient FileSystem fs;
protected final TypedProperties props;
public DFSPathSelector(TypedProperties props, Configuration hadoopConf) {
checkRequiredConfigProperties(
props, Collections.singletonList(DFSPathSelectorConfig.ROOT_INPUT_PATH));
this.props = props;
- this.storage = HoodieStorageUtils.getStorage(
- getStringWithAltKeys(props, DFSPathSelectorConfig.ROOT_INPUT_PATH), HadoopFSUtils.getStorageConf(hadoopConf));
+ this.fs = HadoopFSUtils.getFs(
+ getStringWithAltKeys(props, DFSPathSelectorConfig.ROOT_INPUT_PATH), hadoopConf);
}
/**
@@ -125,19 +124,16 @@ public class DFSPathSelector implements Serializable {
log.info("Root path => " + getStringWithAltKeys(props, DFSPathSelectorConfig.ROOT_INPUT_PATH)
+ " source limit => " + sourceLimit);
long lastCheckpointTime = lastCheckpointStr.map(Long::parseLong).orElse(Long.MIN_VALUE);
- List<StoragePathInfo> eligibleFiles = listEligibleFiles(
- storage, new StoragePath(getStringWithAltKeys(props,
- DFSPathSelectorConfig.ROOT_INPUT_PATH)),
- lastCheckpointTime);
+ List<FileStatus> eligibleFiles = listEligibleFiles(
+ fs, new Path(getStringWithAltKeys(props, DFSPathSelectorConfig.ROOT_INPUT_PATH)), lastCheckpointTime);
// sort them by modification time.
- eligibleFiles.sort(Comparator.comparingLong(StoragePathInfo::getModificationTime));
+ eligibleFiles.sort(Comparator.comparingLong(FileStatus::getModificationTime));
// Filter based on checkpoint & input size, if needed
long currentBytes = 0;
long newCheckpointTime = lastCheckpointTime;
- List<StoragePathInfo> filteredFiles = new ArrayList<>();
- for (StoragePathInfo f : eligibleFiles) {
- if (currentBytes + f.getLength() >= sourceLimit
- && f.getModificationTime() > newCheckpointTime) {
+ List<FileStatus> filteredFiles = new ArrayList<>();
+ for (FileStatus f : eligibleFiles) {
+ if (currentBytes + f.getLen() >= sourceLimit && f.getModificationTime() > newCheckpointTime) {
// we have enough data, we are done
// Also, we've read up to a file with a newer modification time
// so that some files with the same modification time won't be skipped in next read
@@ -145,7 +141,7 @@ public class DFSPathSelector implements Serializable {
}
newCheckpointTime = f.getModificationTime();
- currentBytes += f.getLength();
+ currentBytes += f.getLen();
filteredFiles.add(f);
}
@@ -155,9 +151,7 @@ public class DFSPathSelector implements Serializable {
}
// read the files out.
- String pathStr =
- filteredFiles.stream().map(f -> f.getPath().toString())
- .collect(Collectors.joining(","));
+ String pathStr = filteredFiles.stream().map(f -> f.getPath().toString()).collect(Collectors.joining(","));
return new ImmutablePair<>(Option.ofNullable(pathStr), String.valueOf(newCheckpointTime));
} catch (IOException ioe) {
@@ -168,17 +162,19 @@ public class DFSPathSelector implements Serializable {
/**
* List files recursively, filter out illegible files/directories while doing so.
*/
- protected List<StoragePathInfo> listEligibleFiles(HoodieStorage storage, StoragePath path,
- long lastCheckpointTime) throws IOException {
+ protected List<FileStatus> listEligibleFiles(FileSystem fs, Path path, long lastCheckpointTime) throws IOException {
// skip files/dirs whose names start with (_, ., etc)
- List<StoragePathInfo> pathInfoList = storage.listDirectEntries(path, file ->
+ FileStatus[] statuses = fs.listStatus(path, file ->
IGNORE_FILEPREFIX_LIST.stream().noneMatch(pfx -> file.getName().startsWith(pfx)));
- List<StoragePathInfo> res = new ArrayList<>();
- for (StoragePathInfo pathInfo : pathInfoList) {
- if (pathInfo.isDirectory()) {
- res.addAll(listEligibleFiles(storage, pathInfo.getPath(), lastCheckpointTime));
- } else if (pathInfo.getModificationTime() > lastCheckpointTime && pathInfo.getLength() > 0) {
- res.add(pathInfo);
+ List<FileStatus> res = new ArrayList<>();
+ for (FileStatus status : statuses) {
+ if (status.isDirectory()) {
+ // avoid infinite loop
+ if (!status.isSymlink()) {
+ res.addAll(listEligibleFiles(fs, status.getPath(), lastCheckpointTime));
+ }
+ } else if (status.getModificationTime() > lastCheckpointTime && status.getLen() > 0) {
+ res.add(status);
}
}
return res;
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java
index ab9ccbb8ca7..70acd7ca527 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DatePartitionPathSelector.java
@@ -24,12 +24,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.ImmutablePair;
import org.apache.hudi.common.util.collection.Pair;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
-import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
-import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
import org.apache.hudi.utilities.config.DatePartitionPathSelectorConfig;
import org.apache.hadoop.conf.Configuration;
@@ -136,28 +131,25 @@ public class DatePartitionPathSelector extends DFSPathSelector {
+ currentDate);
long lastCheckpointTime = lastCheckpointStr.map(Long::parseLong).orElse(Long.MIN_VALUE);
HoodieSparkEngineContext context = new HoodieSparkEngineContext(sparkContext);
- StorageConfiguration<?> storageConf = storage.getConf();
+ HadoopStorageConfiguration storageConf = new HadoopStorageConfiguration(fs.getConf());
List<String> prunedPartitionPaths = pruneDatePartitionPaths(
- context, storage, getStringWithAltKeys(props, ROOT_INPUT_PATH),
- currentDate);
+ context, fs, getStringWithAltKeys(props, ROOT_INPUT_PATH), currentDate);
- List<StoragePathInfo> eligibleFiles = context.flatMap(prunedPartitionPaths,
+ List<FileStatus> eligibleFiles = context.flatMap(prunedPartitionPaths,
path -> {
- HoodieStorage storage = HoodieStorageUtils.getStorage(path, storageConf);
- return listEligibleFiles(storage, new StoragePath(path), lastCheckpointTime).stream();
+ FileSystem fs = new Path(path).getFileSystem(storageConf.unwrap());
+ return listEligibleFiles(fs, new Path(path), lastCheckpointTime).stream();
}, partitionsListParallelism);
// sort them by modification time ascending.
- List<StoragePathInfo> sortedEligibleFiles = eligibleFiles.stream()
- .sorted(Comparator.comparingLong(StoragePathInfo::getModificationTime))
- .collect(Collectors.toList());
+ List<FileStatus> sortedEligibleFiles = eligibleFiles.stream()
+ .sorted(Comparator.comparingLong(FileStatus::getModificationTime)).collect(Collectors.toList());
// Filter based on checkpoint & input size, if needed
long currentBytes = 0;
long newCheckpointTime = lastCheckpointTime;
- List<StoragePathInfo> filteredFiles = new ArrayList<>();
- for (StoragePathInfo f : sortedEligibleFiles) {
- if (currentBytes + f.getLength() >= sourceLimit
- && f.getModificationTime() > newCheckpointTime) {
+ List<FileStatus> filteredFiles = new ArrayList<>();
+ for (FileStatus f : sortedEligibleFiles) {
+ if (currentBytes + f.getLen() >= sourceLimit && f.getModificationTime() > newCheckpointTime) {
// we have enough data, we are done
// Also, we've read up to a file with a newer modification time
// so that some files with the same modification time won't be skipped in next read
@@ -165,7 +157,7 @@ public class DatePartitionPathSelector extends DFSPathSelector {
}
newCheckpointTime = f.getModificationTime();
- currentBytes += f.getLength();
+ currentBytes += f.getLen();
filteredFiles.add(f);
}
@@ -175,9 +167,7 @@ public class DatePartitionPathSelector extends DFSPathSelector {
}
// read the files out.
- String pathStr =
- filteredFiles.stream().map(f -> f.getPath().toString())
- .collect(Collectors.joining(","));
+ String pathStr = filteredFiles.stream().map(f -> f.getPath().toString()).collect(Collectors.joining(","));
return new ImmutablePair<>(Option.ofNullable(pathStr), String.valueOf(newCheckpointTime));
}
@@ -186,25 +176,21 @@ public class DatePartitionPathSelector extends DFSPathSelector {
* Prunes date level partitions to last few days configured by 'NUM_PREV_DAYS_TO_LIST' from
* 'CURRENT_DATE'. Parallelizes listing by leveraging HoodieSparkEngineContext's methods.
*/
- public List<String> pruneDatePartitionPaths(HoodieSparkEngineContext context,
- HoodieStorage storage,
- String rootPath, LocalDate currentDate) {
+ public List<String> pruneDatePartitionPaths(HoodieSparkEngineContext context, FileSystem fs, String rootPath, LocalDate currentDate) {
List<String> partitionPaths = new ArrayList<>();
// get all partition paths before date partition level
partitionPaths.add(rootPath);
if (datePartitionDepth <= 0) {
return partitionPaths;
}
- StorageConfiguration<Configuration> storageConf = HadoopFSUtils.getStorageConfWithCopy(
- ((FileSystem) storage.getFileSystem()).getConf());
+ HadoopStorageConfiguration storageConf = new HadoopStorageConfiguration(fs.getConf());
for (int i = 0; i < datePartitionDepth; i++) {
partitionPaths = context.flatMap(partitionPaths, path -> {
Path subDir = new Path(path);
FileSystem fileSystem = subDir.getFileSystem(storageConf.unwrap());
// skip files/dirs whose names start with (_, ., etc)
FileStatus[] statuses = fileSystem.listStatus(subDir,
- file -> IGNORE_FILEPREFIX_LIST.stream()
- .noneMatch(pfx -> file.getName().startsWith(pfx)));
+ file -> IGNORE_FILEPREFIX_LIST.stream().noneMatch(pfx -> file.getName().startsWith(pfx)));
List<String> res = new ArrayList<>();
for (FileStatus status : statuses) {
res.add(status.getPath().toString());
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieMultiTableStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieMultiTableStreamer.java
index f1116150be3..a637f7fbbff 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieMultiTableStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieMultiTableStreamer.java
@@ -29,7 +29,6 @@ import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.hive.HiveSyncTool;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.sync.common.HoodieSyncConfig;
import org.apache.hudi.utilities.IdentitySplitter;
import org.apache.hudi.utilities.UtilHelpers;
@@ -90,7 +89,7 @@ public class HoodieMultiTableStreamer {
FileSystem fs = HadoopFSUtils.getFs(commonPropsFile, jssc.hadoopConfiguration());
configFolder = configFolder.charAt(configFolder.length() - 1) == '/' ? configFolder.substring(0, configFolder.length() - 1) : configFolder;
checkIfPropsFileAndConfigFolderExist(commonPropsFile, configFolder, fs);
- TypedProperties commonProperties = UtilHelpers.readConfig(fs.getConf(), new StoragePath(commonPropsFile), new ArrayList<String>()).getProps();
+ TypedProperties commonProperties = UtilHelpers.readConfig(fs.getConf(), new Path(commonPropsFile), new ArrayList<String>()).getProps();
//get the tables to be ingested and their corresponding config files from this properties instance
populateTableExecutionContextList(commonProperties, configFolder, fs, config);
}
@@ -131,7 +130,7 @@ public class HoodieMultiTableStreamer {
String configFilePath = getStringWithAltKeys(properties, configProp, oldConfigProp,
Helpers.getDefaultConfigFilePath(configFolder, database, currentTable));
checkIfTableConfigFileExists(configFolder, fs, configFilePath);
- TypedProperties tableProperties = UtilHelpers.readConfig(fs.getConf(), new StoragePath(configFilePath), new ArrayList<>()).getProps();
+ TypedProperties tableProperties = UtilHelpers.readConfig(fs.getConf(), new Path(configFilePath), new ArrayList<>()).getProps();
properties.forEach((k, v) -> {
if (tableProperties.get(k) == null) {
tableProperties.setProperty(k.toString(), v.toString());
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
index 4ea84ff7a5e..4fe25870201 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
@@ -74,6 +74,7 @@ import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;
@@ -178,7 +179,7 @@ public class HoodieStreamer implements Serializable {
} else if (cfg.propsFilePath.equals(Config.DEFAULT_DFS_SOURCE_PROPERTIES)) {
hoodieConfig.setAll(UtilHelpers.getConfig(cfg.configs).getProps());
} else {
- hoodieConfig.setAll(readConfig(hadoopConf, new StoragePath(cfg.propsFilePath), cfg.configs).getProps());
+ hoodieConfig.setAll(readConfig(hadoopConf, new Path(cfg.propsFilePath), cfg.configs).getProps());
}
// set any configs that Deltastreamer has to override explicitly
@@ -447,7 +448,7 @@ public class HoodieStreamer implements Serializable {
public static TypedProperties getProps(Configuration conf, Config cfg) {
return cfg.propsFilePath.isEmpty()
? buildProperties(cfg.configs)
- : readConfig(conf, new StoragePath(cfg.propsFilePath), cfg.configs).getProps();
+ : readConfig(conf, new Path(cfg.propsFilePath), cfg.configs).getProps();
}
@Override
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDFSPathSelectorCommonMethods.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDFSPathSelectorCommonMethods.java
index a31938c439b..684261cab84 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDFSPathSelectorCommonMethods.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDFSPathSelectorCommonMethods.java
@@ -23,10 +23,11 @@ import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.collection.Pair;
-import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.params.ParameterizedTest;
@@ -46,7 +47,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestDFSPathSelectorCommonMethods extends HoodieSparkClientTestHarness {
TypedProperties props;
- StoragePath inputPath;
+ Path inputPath;
@BeforeEach
void setUp() {
@@ -56,7 +57,7 @@ public class TestDFSPathSelectorCommonMethods extends HoodieSparkClientTestHarne
props = new TypedProperties();
props.setProperty(ROOT_INPUT_PATH.key(), basePath);
props.setProperty(PARTITIONS_LIST_PARALLELISM.key(), "1");
- inputPath = new StoragePath(basePath);
+ inputPath = new Path(basePath);
}
@AfterEach
@@ -72,7 +73,8 @@ public class TestDFSPathSelectorCommonMethods extends HoodieSparkClientTestHarne
createBaseFile(basePath, "p1", "000", ".foo2", 1);
createBaseFile(basePath, "p1", "000", "_foo3", 1);
- List<StoragePathInfo> eligibleFiles = selector.listEligibleFiles(storage, inputPath, 0);
+ List<FileStatus> eligibleFiles = selector.listEligibleFiles(
+ (FileSystem) storage.getFileSystem(), inputPath, 0);
assertEquals(1, eligibleFiles.size());
assertTrue(eligibleFiles.get(0).getPath().getName().startsWith("foo1"));
}
@@ -85,7 +87,8 @@ public class TestDFSPathSelectorCommonMethods extends HoodieSparkClientTestHarne
createBaseFile(basePath, "p1", "000", "foo2", 0);
createBaseFile(basePath, "p1", "000", "foo3", 0);
- List<StoragePathInfo> eligibleFiles = selector.listEligibleFiles(storage, inputPath, 0);
+ List<FileStatus> eligibleFiles = selector.listEligibleFiles(
+ (FileSystem) storage.getFileSystem(), inputPath, 0);
assertEquals(1, eligibleFiles.size());
assertTrue(eligibleFiles.get(0).getPath().getName().startsWith("foo1"));
}
@@ -98,8 +101,8 @@ public class TestDFSPathSelectorCommonMethods extends HoodieSparkClientTestHarne
createBaseFile(basePath, "p1", "000", "foo2", 1);
createBaseFile(basePath, "p1", "000", "foo3", 1);
- List<StoragePathInfo> eligibleFiles =
- selector.listEligibleFiles(storage, inputPath, Long.MAX_VALUE);
+ List<FileStatus> eligibleFiles = selector.listEligibleFiles(
+ (FileSystem) storage.getFileSystem(), inputPath, Long.MAX_VALUE);
assertEquals(0, eligibleFiles.size());
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDatePartitionPathSelector.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDatePartitionPathSelector.java
index 439f01600be..509463c58aa 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDatePartitionPathSelector.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestDatePartitionPathSelector.java
@@ -24,6 +24,7 @@ import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
import org.apache.hudi.utilities.testutils.UtilitiesTestBase;
+import org.apache.hadoop.fs.FileSystem;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -215,8 +216,8 @@ public class TestDatePartitionPathSelector extends HoodieSparkClientTestHarness
createParentDirsBeforeDatePartitions(root, generateRandomStrings(), totalDepthBeforeDatePartitions, leafDirs);
createDatePartitionsWithFiles(leafDirs, isHiveStylePartition, dateFormat);
- List<String> paths = pathSelector.pruneDatePartitionPaths(context, storage, root.toString(),
- LocalDate.parse(currentDate));
+ List<String> paths = pathSelector.pruneDatePartitionPaths(
+ context, (FileSystem) storage.getFileSystem(), root.toString(), LocalDate.parse(currentDate));
assertEquals(expectedNumFiles, paths.size());
}
}
(hudi) 05/07: [HUDI-7792] Bump h2 from 1.4.200 to 2.2.220 (#11296)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch release-0.15.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 2e763c9ec8ee8d729a3f2cc4ec19d50555b5f222
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Fri May 24 23:02:02 2024 -0700
[HUDI-7792] Bump h2 from 1.4.200 to 2.2.220 (#11296)
---
hudi-platform-service/hudi-metaserver/pom.xml | 2 +-
packaging/hudi-metaserver-server-bundle/pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/hudi-platform-service/hudi-metaserver/pom.xml b/hudi-platform-service/hudi-metaserver/pom.xml
index 52ccb68b25f..bf29a45080c 100644
--- a/hudi-platform-service/hudi-metaserver/pom.xml
+++ b/hudi-platform-service/hudi-metaserver/pom.xml
@@ -32,7 +32,7 @@
<properties>
<main.basedir>${project.parent.basedir}</main.basedir>
- <h2.version>1.4.200</h2.version>
+ <h2.version>2.2.220</h2.version>
<!-- Thrift related properties -->
<thrift.home>/usr/local</thrift.home>
<thrift.install.env>docker</thrift.install.env>
diff --git a/packaging/hudi-metaserver-server-bundle/pom.xml b/packaging/hudi-metaserver-server-bundle/pom.xml
index 162468c69bd..9ffcc24ebb2 100644
--- a/packaging/hudi-metaserver-server-bundle/pom.xml
+++ b/packaging/hudi-metaserver-server-bundle/pom.xml
@@ -30,7 +30,7 @@
<properties>
<main.basedir>${project.parent.basedir}</main.basedir>
- <h2.version>1.4.200</h2.version>
+ <h2.version>2.2.220</h2.version>
</properties>
<dependencies>
(hudi) 01/07: [HUDI-7785] Keep public APIs in utilities module the same as before HoodieStorage abstraction (#11280)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch release-0.15.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit b71da938ac5ce7913965a2e7dfb865cc40e901b5
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Thu May 23 23:25:27 2024 -0700
[HUDI-7785] Keep public APIs in utilities module the same as before HoodieStorage abstraction (#11280)
---
.../hudi/utilities/deltastreamer/DeltaSync.java | 3 +-
.../deltastreamer/HoodieDeltaStreamer.java | 5 +-
.../utilities/streamer/BaseErrorTableWriter.java | 13 ++--
.../hudi/utilities/streamer/ErrorTableUtils.java | 5 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 41 ++++++-------
.../apache/hudi/utilities/streamer/StreamSync.java | 8 +--
...TestHoodieDeltaStreamerSchemaEvolutionBase.java | 6 +-
.../utilities/sources/TestJsonKafkaSource.java | 2 +-
.../utilities/streamer/TestErrorTableUtils.java | 70 ++++++++++++++++++++++
9 files changed, 111 insertions(+), 42 deletions(-)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
index c8a1b47b9fb..24fa9b97051 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
@@ -23,7 +23,6 @@ import org.apache.hudi.client.SparkRDDWriteClient;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.utilities.schema.SchemaProvider;
import org.apache.hudi.utilities.streamer.DefaultStreamContext;
import org.apache.hudi.utilities.streamer.HoodieStreamer;
@@ -53,6 +52,6 @@ public class DeltaSync extends StreamSync {
TypedProperties props, HoodieSparkEngineContext hoodieSparkContext, FileSystem fs, Configuration conf,
Function<SparkRDDWriteClient, Boolean> onInitializingHoodieWriteClient) throws IOException {
super(cfg, sparkSession, props, hoodieSparkContext,
- new HoodieHadoopStorage(fs), conf, onInitializingHoodieWriteClient, new DefaultStreamContext(schemaProvider, Option.empty()));
+ fs, conf, onInitializingHoodieWriteClient, new DefaultStreamContext(schemaProvider, Option.empty()));
}
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
index 6c5cca9888e..8d941886a08 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
@@ -21,7 +21,6 @@ package org.apache.hudi.utilities.deltastreamer;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.utilities.streamer.HoodieStreamer;
import org.apache.hadoop.conf.Configuration;
@@ -51,7 +50,7 @@ public class HoodieDeltaStreamer extends HoodieStreamer {
JavaSparkContext jssc,
FileSystem fs,
Configuration conf) throws IOException {
- super(cfg, jssc, new HoodieHadoopStorage(fs), conf);
+ super(cfg, jssc, fs, conf);
}
public HoodieDeltaStreamer(Config cfg,
@@ -59,7 +58,7 @@ public class HoodieDeltaStreamer extends HoodieStreamer {
FileSystem fs,
Configuration conf,
Option<TypedProperties> propsOverride) throws IOException {
- super(cfg, jssc, new HoodieHadoopStorage(fs), conf, propsOverride);
+ super(cfg, jssc, fs, conf, propsOverride);
}
@Deprecated
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/BaseErrorTableWriter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/BaseErrorTableWriter.java
index b9d18dbd916..5c25d68d2c4 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/BaseErrorTableWriter.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/BaseErrorTableWriter.java
@@ -19,13 +19,16 @@
package org.apache.hudi.utilities.streamer;
+import org.apache.hudi.ApiMaturityLevel;
+import org.apache.hudi.PublicAPIClass;
+import org.apache.hudi.PublicAPIMethod;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.VisibleForTesting;
-import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.SparkSession;
@@ -40,6 +43,7 @@ import java.io.Serializable;
*
* The writer can use the configs defined in HoodieErrorTableConfig to manage the error table.
*/
+@PublicAPIClass(maturity = ApiMaturityLevel.EVOLVING)
public abstract class BaseErrorTableWriter<T extends ErrorEvent> implements Serializable {
// The column name passed to Spark for option `columnNameOfCorruptRecord`. The record
@@ -47,8 +51,7 @@ public abstract class BaseErrorTableWriter<T extends ErrorEvent> implements Seri
public static String ERROR_TABLE_CURRUPT_RECORD_COL_NAME = "_corrupt_record";
public BaseErrorTableWriter(HoodieStreamer.Config cfg, SparkSession sparkSession,
- TypedProperties props,
- HoodieSparkEngineContext hoodieSparkContext, HoodieStorage storage) {
+ TypedProperties props, HoodieSparkEngineContext hoodieSparkContext, FileSystem fileSystem) {
}
/**
@@ -57,18 +60,20 @@ public abstract class BaseErrorTableWriter<T extends ErrorEvent> implements Seri
*
* @param errorEvent Input error event RDD
*/
+ @PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
public abstract void addErrorEvents(JavaRDD<T> errorEvent);
/**
* Fetches the error events RDD processed by the writer so far. This is a test API.
*/
@VisibleForTesting
+ @PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
public abstract Option<JavaRDD<HoodieAvroRecord>> getErrorEvents(String baseTableInstantTime, Option<String> commitedInstantTime);
/**
* This API is called to commit the error events (failed Hoodie Records) processed by the writer so far.
* These records are committed to a error table.
*/
+ @PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
public abstract boolean upsertAndCommit(String baseTableInstantTime, Option<String> commitedInstantTime);
-
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorTableUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorTableUtils.java
index fce14d18807..9d9de91fbba 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorTableUtils.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorTableUtils.java
@@ -28,7 +28,6 @@ import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.HoodieErrorTableConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieValidationException;
-import org.apache.hudi.storage.HoodieStorage;
import org.apache.hadoop.fs.FileSystem;
import org.apache.spark.sql.Dataset;
@@ -48,7 +47,7 @@ public final class ErrorTableUtils {
SparkSession sparkSession,
TypedProperties props,
HoodieSparkEngineContext hoodieSparkContext,
- HoodieStorage storage) {
+ FileSystem fileSystem) {
String errorTableWriterClass = props.getString(ERROR_TABLE_WRITE_CLASS.key());
ValidationUtils.checkState(!StringUtils.isNullOrEmpty(errorTableWriterClass),
"Missing error table config " + ERROR_TABLE_WRITE_CLASS);
@@ -65,7 +64,7 @@ public final class ErrorTableUtils {
try {
return Option.of((BaseErrorTableWriter) ReflectionUtils.getClass(errorTableWriterClass)
.getConstructor(argClassArr)
- .newInstance(cfg, sparkSession, props, hoodieSparkContext, storage));
+ .newInstance(cfg, sparkSession, props, hoodieSparkContext, fileSystem));
} catch (NoSuchMethodException | InvocationTargetException | InstantiationException
| IllegalAccessException e) {
throw new HoodieException(errMsg, e);
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
index 5af958d108b..4ea84ff7a5e 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
@@ -58,8 +58,8 @@ import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.hive.HiveSyncTool;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.utilities.HiveIncrementalPuller;
import org.apache.hudi.utilities.IdentitySplitter;
import org.apache.hudi.utilities.UtilHelpers;
@@ -132,26 +132,24 @@ public class HoodieStreamer implements Serializable {
public static final String STREAMSYNC_POOL_NAME = "hoodiedeltasync";
public HoodieStreamer(Config cfg, JavaSparkContext jssc) throws IOException {
- this(cfg, jssc,
- HoodieStorageUtils.getStorage(cfg.targetBasePath, HadoopFSUtils.getStorageConf(jssc.hadoopConfiguration())),
+ this(cfg, jssc, HadoopFSUtils.getFs(cfg.targetBasePath, jssc.hadoopConfiguration()),
jssc.hadoopConfiguration(), Option.empty());
}
public HoodieStreamer(Config cfg, JavaSparkContext jssc, Option<TypedProperties> props) throws IOException {
- this(cfg, jssc,
- HoodieStorageUtils.getStorage(cfg.targetBasePath, HadoopFSUtils.getStorageConf(jssc.hadoopConfiguration())),
+ this(cfg, jssc, HadoopFSUtils.getFs(cfg.targetBasePath, jssc.hadoopConfiguration()),
jssc.hadoopConfiguration(), props);
}
- public HoodieStreamer(Config cfg, JavaSparkContext jssc, HoodieStorage storage, Configuration conf) throws IOException {
- this(cfg, jssc, storage, conf, Option.empty());
+ public HoodieStreamer(Config cfg, JavaSparkContext jssc, FileSystem fs, Configuration conf) throws IOException {
+ this(cfg, jssc, fs, conf, Option.empty());
}
- public HoodieStreamer(Config cfg, JavaSparkContext jssc, HoodieStorage storage, Configuration conf, Option<TypedProperties> propsOverride) throws IOException {
- this(cfg, jssc, storage, conf, propsOverride, Option.empty());
+ public HoodieStreamer(Config cfg, JavaSparkContext jssc, FileSystem fs, Configuration conf, Option<TypedProperties> propsOverride) throws IOException {
+ this(cfg, jssc, fs, conf, propsOverride, Option.empty());
}
- public HoodieStreamer(Config cfg, JavaSparkContext jssc, HoodieStorage storage, Configuration conf,
+ public HoodieStreamer(Config cfg, JavaSparkContext jssc, FileSystem fs, Configuration conf,
Option<TypedProperties> propsOverride, Option<SourceProfileSupplier> sourceProfileSupplier) throws IOException {
this.properties = combineProperties(cfg, propsOverride, jssc.hadoopConfiguration());
if (cfg.initialCheckpointProvider != null && cfg.checkpoint == null) {
@@ -163,11 +161,10 @@ public class HoodieStreamer implements Serializable {
this.cfg = cfg;
this.bootstrapExecutor = Option.ofNullable(
- cfg.runBootstrap ? new BootstrapExecutor(
- cfg, jssc, (FileSystem) storage.getFileSystem(), conf, this.properties) : null);
+ cfg.runBootstrap ? new BootstrapExecutor(cfg, jssc, fs, conf, this.properties) : null);
HoodieSparkEngineContext sparkEngineContext = new HoodieSparkEngineContext(jssc);
this.ingestionService = Option.ofNullable(
- cfg.runBootstrap ? null : new StreamSyncService(cfg, sparkEngineContext, storage, conf, Option.ofNullable(this.properties), sourceProfileSupplier));
+ cfg.runBootstrap ? null : new StreamSyncService(cfg, sparkEngineContext, fs, conf, Option.ofNullable(this.properties), sourceProfileSupplier));
}
private static TypedProperties combineProperties(Config cfg, Option<TypedProperties> propsOverride, Configuration hadoopConf) {
@@ -672,14 +669,14 @@ public class HoodieStreamer implements Serializable {
private final Option<ConfigurationHotUpdateStrategy> configurationHotUpdateStrategyOpt;
public StreamSyncService(Config cfg, HoodieSparkEngineContext hoodieSparkContext,
- HoodieStorage storage, Configuration conf,
+ FileSystem fs, Configuration conf,
Option<TypedProperties> properties, Option<SourceProfileSupplier> sourceProfileSupplier) throws IOException {
super(HoodieIngestionConfig.newBuilder()
.isContinuous(cfg.continuousMode)
.withMinSyncInternalSeconds(cfg.minSyncIntervalSeconds).build());
this.cfg = cfg;
this.hoodieSparkContext = hoodieSparkContext;
- this.storage = storage;
+ this.storage = new HoodieHadoopStorage(fs);
this.hiveConf = conf;
this.sparkSession = SparkSession.builder().config(hoodieSparkContext.getConf()).getOrCreate();
this.asyncCompactService = Option.empty();
@@ -732,20 +729,19 @@ public class HoodieStreamer implements Serializable {
props, hoodieSparkContext.jsc(), cfg.transformerClassNames);
streamSync = new StreamSync(cfg, sparkSession, props, hoodieSparkContext,
- this.storage, conf, this::onInitializingWriteClient, new DefaultStreamContext(schemaProvider, sourceProfileSupplier));
-
+ fs, conf, this::onInitializingWriteClient, new DefaultStreamContext(schemaProvider, sourceProfileSupplier));
}
public StreamSyncService(HoodieStreamer.Config cfg,
- HoodieSparkEngineContext hoodieSparkContext, HoodieStorage storage,
+ HoodieSparkEngineContext hoodieSparkContext, FileSystem fs,
Configuration conf)
throws IOException {
- this(cfg, hoodieSparkContext, storage, conf, Option.empty(), Option.empty());
+ this(cfg, hoodieSparkContext, fs, conf, Option.empty(), Option.empty());
}
- public StreamSyncService(HoodieStreamer.Config cfg, HoodieSparkEngineContext hoodieSparkContext, HoodieStorage storage, Configuration conf, Option<TypedProperties> properties)
+ public StreamSyncService(HoodieStreamer.Config cfg, HoodieSparkEngineContext hoodieSparkContext, FileSystem fs, Configuration conf, Option<TypedProperties> properties)
throws IOException {
- this(cfg, hoodieSparkContext, storage, conf, properties, Option.empty());
+ this(cfg, hoodieSparkContext, fs, conf, properties, Option.empty());
}
private void initializeTableTypeAndBaseFileFormat() {
@@ -760,7 +756,8 @@ public class HoodieStreamer implements Serializable {
streamSync.close();
}
streamSync = new StreamSync(cfg, sparkSession, props, hoodieSparkContext,
- storage, hiveConf, this::onInitializingWriteClient, new DefaultStreamContext(schemaProvider, Option.empty()));
+ (FileSystem) storage.getFileSystem(), hiveConf, this::onInitializingWriteClient,
+ new DefaultStreamContext(schemaProvider, Option.empty()));
}
@Override
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
index 20e530c2ee7..d5405645782 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
@@ -292,18 +292,18 @@ public class StreamSync implements Serializable, Closeable {
TypedProperties props, JavaSparkContext jssc, FileSystem fs, Configuration conf,
Function<SparkRDDWriteClient, Boolean> onInitializingHoodieWriteClient) throws IOException {
this(cfg, sparkSession, props, new HoodieSparkEngineContext(jssc),
- new HoodieHadoopStorage(fs), conf, onInitializingHoodieWriteClient,
+ fs, conf, onInitializingHoodieWriteClient,
new DefaultStreamContext(schemaProvider, Option.empty()));
}
public StreamSync(HoodieStreamer.Config cfg, SparkSession sparkSession,
TypedProperties props, HoodieSparkEngineContext hoodieSparkContext,
- HoodieStorage storage, Configuration conf,
+ FileSystem fs, Configuration conf,
Function<SparkRDDWriteClient, Boolean> onInitializingHoodieWriteClient, StreamContext streamContext) throws IOException {
this.cfg = cfg;
this.hoodieSparkContext = hoodieSparkContext;
this.sparkSession = sparkSession;
- this.storage = storage;
+ this.storage = new HoodieHadoopStorage(fs);
this.onInitializingHoodieWriteClient = onInitializingHoodieWriteClient;
this.props = props;
this.userProvidedSchemaProvider = streamContext.getSchemaProvider();
@@ -319,7 +319,7 @@ public class StreamSync implements Serializable, Closeable {
this.hoodieMetrics = new HoodieMetrics(hoodieWriteConfig, storage.getConf());
if (props.getBoolean(ERROR_TABLE_ENABLED.key(), ERROR_TABLE_ENABLED.defaultValue())) {
this.errorTableWriter = ErrorTableUtils.getErrorTableWriter(
- cfg, sparkSession, props, hoodieSparkContext, storage);
+ cfg, sparkSession, props, hoodieSparkContext, fs);
this.errorWriteFailureStrategy = ErrorTableUtils.getErrorWriteFailureStrategy(props);
}
refreshTimeline();
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
index c6f2afc2ef7..e4670799830 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
@@ -32,7 +32,6 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieClusteringConfig;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieErrorTableConfig;
-import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.utilities.schema.FilebasedSchemaProvider;
import org.apache.hudi.utilities.schema.SchemaProvider;
import org.apache.hudi.utilities.sources.AvroKafkaSource;
@@ -42,6 +41,7 @@ import org.apache.hudi.utilities.streamer.HoodieStreamer;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
@@ -327,8 +327,8 @@ public class TestHoodieDeltaStreamerSchemaEvolutionBase extends HoodieDeltaStrea
public static Map<String,Option<JavaRDD>> commited = new HashMap<>();
public TestErrorTable(HoodieStreamer.Config cfg, SparkSession sparkSession, TypedProperties props, HoodieSparkEngineContext hoodieSparkContext,
- HoodieStorage storage) {
- super(cfg, sparkSession, props, hoodieSparkContext, storage);
+ FileSystem fileSystem) {
+ super(cfg, sparkSession, props, hoodieSparkContext, fileSystem);
}
@Override
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJsonKafkaSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJsonKafkaSource.java
index 92238721fcd..90d8543a5db 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJsonKafkaSource.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJsonKafkaSource.java
@@ -296,7 +296,7 @@ public class TestJsonKafkaSource extends BaseTestKafkaSource {
private BaseErrorTableWriter getAnonymousErrorTableWriter(TypedProperties props) {
return new BaseErrorTableWriter<ErrorEvent<String>>(new HoodieDeltaStreamer.Config(),
- spark(), props, new HoodieSparkEngineContext(jsc()), hoodieStorage()) {
+ spark(), props, new HoodieSparkEngineContext(jsc()), fs()) {
List<JavaRDD<HoodieAvroRecord>> errorEvents = new LinkedList();
@Override
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestErrorTableUtils.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestErrorTableUtils.java
new file mode 100644
index 00000000000..1a3af25cd59
--- /dev/null
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestErrorTableUtils.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.utilities.streamer;
+
+import org.apache.hudi.client.common.HoodieSparkEngineContext;
+import org.apache.hudi.common.config.HoodieConfig;
+import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamerSchemaEvolutionBase.TestErrorTable;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Tests {@link ErrorTableUtils}.
+ */
+public class TestErrorTableUtils {
+ @Test
+ public void testGetErrorTableWriter() {
+ SparkSession sparkSession = Mockito.mock(SparkSession.class);
+ HoodieSparkEngineContext sparkContext = Mockito.mock(HoodieSparkEngineContext.class);
+ FileSystem fileSystem = Mockito.mock(FileSystem.class);
+
+ TypedProperties props = new TypedProperties();
+ // No error table writer config
+ assertThrows(IllegalArgumentException.class,
+ () -> ErrorTableUtils.getErrorTableWriter(
+ new HoodieStreamer.Config(), sparkSession, props, sparkContext, fileSystem));
+
+ // Empty error table writer config
+ props.put("hoodie.errortable.write.class", StringUtils.EMPTY_STRING);
+ assertThrows(IllegalStateException.class,
+ () -> ErrorTableUtils.getErrorTableWriter(
+ new HoodieStreamer.Config(), sparkSession, props, sparkContext, fileSystem));
+
+ // Proper error table writer config
+ props.put("hoodie.errortable.write.class", TestErrorTable.class.getName());
+ assertTrue(ErrorTableUtils.getErrorTableWriter(
+ new HoodieStreamer.Config(), sparkSession, props, sparkContext, fileSystem).get() instanceof TestErrorTable);
+
+ // Wrong error table writer config
+ props.put("hoodie.errortable.write.class", HoodieConfig.class.getName());
+ assertThrows(HoodieException.class,
+ () -> ErrorTableUtils.getErrorTableWriter(
+ new HoodieStreamer.Config(), sparkSession, props, sparkContext, fileSystem));
+ }
+}
(hudi) 07/07: [HUDI-7794] Bump org.apache.hive:hive-service from 2.3.1 to 2.3.4 (#11298)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch release-0.15.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit b6078994449cd4cae0ed8b94efb338ac4d40cada
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Sat May 25 00:42:15 2024 -0700
[HUDI-7794] Bump org.apache.hive:hive-service from 2.3.1 to 2.3.4 (#11298)
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
---
packaging/hudi-flink-bundle/pom.xml | 2 +-
pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/packaging/hudi-flink-bundle/pom.xml b/packaging/hudi-flink-bundle/pom.xml
index 19b34d15ae9..41b80e7f58d 100644
--- a/packaging/hudi-flink-bundle/pom.xml
+++ b/packaging/hudi-flink-bundle/pom.xml
@@ -37,7 +37,7 @@
<javax.servlet.version>3.1.0</javax.servlet.version>
<!-- override to be same with flink 1.15.x -->
<parquet.version>${flink.format.parquet.version}</parquet.version>
- <hive.version>2.3.1</hive.version>
+ <hive.version>2.3.4</hive.version>
<thrift.version>0.9.3</thrift.version>
</properties>
diff --git a/pom.xml b/pom.xml
index a3ba096a5e5..95a055d541d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -120,7 +120,7 @@
<joda.version>2.9.9</joda.version>
<hadoop.version>2.10.2</hadoop.version>
<hive.groupid>org.apache.hive</hive.groupid>
- <hive.version>2.3.1</hive.version>
+ <hive.version>2.3.4</hive.version>
<hive.parquet.version>1.10.1</hive.parquet.version>
<hive.avro.version>1.8.2</hive.avro.version>
<presto.version>0.273</presto.version>
(hudi) 03/07: [HUDI-7788] Fixing exception handling in AverageRecordSizeUtils (#11290)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch release-0.15.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit b350a26cabf8fe0f3e04def635709f3f5120869c
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Fri May 24 14:17:48 2024 -0700
[HUDI-7788] Fixing exception handling in AverageRecordSizeUtils (#11290)
---
.../table/action/commit/AverageRecordSizeUtils.java | 5 ++---
.../action/commit/TestAverageRecordSizeUtils.java | 20 ++++++++++++++++++++
2 files changed, 22 insertions(+), 3 deletions(-)
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java
index 9d9408e173b..b0cd1ca5f7e 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java
@@ -29,7 +29,6 @@ import org.apache.hudi.storage.StoragePath;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.util.Iterator;
import java.util.concurrent.atomic.AtomicLong;
@@ -80,9 +79,9 @@ public class AverageRecordSizeUtils {
break;
}
}
- } catch (IOException ioe) {
+ } catch (Throwable t) {
// make this fail safe.
- LOG.error("Error trying to compute average bytes/record ", ioe);
+ LOG.error("Error trying to compute average bytes/record ", t);
}
}
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java
index 5db8c978b65..2addb0296f6 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java
@@ -29,6 +29,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -44,6 +45,7 @@ import java.util.stream.Stream;
import static org.apache.hudi.common.model.HoodieFileFormat.HOODIE_LOG;
import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET;
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
+import static org.apache.hudi.config.HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -91,6 +93,24 @@ public class TestAverageRecordSizeUtils {
assertEquals(expectedSize, AverageRecordSizeUtils.averageBytesPerRecord(mockTimeline, writeConfig));
}
+ @Test
+ public void testErrorHandling() {
+ int recordSize = 10000;
+ HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder()
+ .withProps(Collections.singletonMap(COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), String.valueOf(recordSize)))
+ .build(false);
+ HoodieDefaultTimeline commitsTimeline = new HoodieDefaultTimeline();
+ List<HoodieInstant> instants = Collections.singletonList(
+ new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "1"));
+
+ when(mockTimeline.getInstants()).thenReturn(instants);
+ when(mockTimeline.getReverseOrderedInstants()).then(i -> instants.stream());
+ // Simulate a case where the instant details are absent
+ commitsTimeline.setInstants(new ArrayList<>());
+
+ assertEquals(recordSize, AverageRecordSizeUtils.averageBytesPerRecord(mockTimeline, writeConfig));
+ }
+
private static String getBaseFileName(String instantTime) {
String fileName = UUID.randomUUID().toString();
return FSUtils.makeBaseFileName(instantTime, TEST_WRITE_TOKEN, fileName, PARQUET.getFileExtension());
(hudi) 04/07: [HUDI-7776] Simplify HoodieStorage instance fetching (#11292)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch release-0.15.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 77b1ded0209519413d09f468aa7c9bc1d8b53c67
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Fri May 24 23:01:03 2024 -0700
[HUDI-7776] Simplify HoodieStorage instance fetching (#11292)
---
.../hudi/cli/commands/ArchivedCommitsCommand.java | 2 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 22 ++---
.../apache/hudi/cli/commands/RepairsCommand.java | 2 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 2 +-
.../apache/hudi/client/HoodieTimelineArchiver.java | 2 +-
.../hudi/client/heartbeat/HeartbeatUtils.java | 5 +-
.../hudi/client/transaction/lock/LockManager.java | 3 +-
.../lock/metrics/HoodieLockMetrics.java | 14 +--
.../org/apache/hudi/index/HoodieIndexUtils.java | 7 +-
.../hudi/index/bloom/HoodieGlobalBloomIndex.java | 2 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 6 +-
.../hudi/index/simple/HoodieGlobalSimpleIndex.java | 2 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 2 +-
.../hudi/io/HoodieKeyLocationFetchHandle.java | 4 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieReadHandle.java | 8 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 8 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 6 +-
.../java/org/apache/hudi/table/HoodieTable.java | 10 +-
.../table/action/clean/CleanActionExecutor.java | 6 +-
.../hudi/table/action/clean/CleanPlanner.java | 4 +-
.../PartitionAwareClusteringPlanStrategy.java | 3 +-
.../action/commit/BaseCommitActionExecutor.java | 2 +-
.../table/action/commit/HoodieMergeHelper.java | 13 +--
.../compact/RunCompactionActionExecutor.java | 2 +-
.../BaseHoodieCompactionPlanGenerator.java | 2 +-
.../index/RecordBasedIndexingCatchupTask.java | 3 +-
.../table/action/index/RunIndexActionExecutor.java | 6 +-
.../action/index/ScheduleIndexActionExecutor.java | 2 +-
.../action/restore/BaseRestoreActionExecutor.java | 2 +-
.../rollback/BaseRollbackActionExecutor.java | 4 +-
.../rollback/ListingBasedRollbackStrategy.java | 2 +-
.../action/savepoint/SavepointActionExecutor.java | 3 +-
.../hudi/table/marker/DirectWriteMarkers.java | 2 +-
.../table/marker/MarkerBasedRollbackUtils.java | 2 +-
.../hudi/avro/TestHoodieAvroParquetWriter.java | 16 ++--
.../hudi/client/utils/TestCommitMetadataUtils.java | 2 +-
.../hudi/metrics/TestHoodieConsoleMetrics.java | 4 +-
.../hudi/metrics/TestHoodieGraphiteMetrics.java | 4 +-
.../apache/hudi/metrics/TestHoodieJmxMetrics.java | 4 +-
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 4 +-
.../datadog/TestDatadogMetricsReporter.java | 10 +-
.../org/apache/hudi/metrics/m3/TestM3Metrics.java | 21 ++---
.../metrics/prometheus/TestPrometheusReporter.java | 6 +-
.../prometheus/TestPushGateWayReporter.java | 6 +-
.../GenericRecordValidationTestUtils.java | 7 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 2 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 7 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 2 +-
.../FlinkDeletePartitionCommitActionExecutor.java | 2 +-
.../index/bloom/TestFlinkHoodieBloomIndex.java | 3 +-
.../run/strategy/JavaExecutionStrategy.java | 7 +-
.../JavaHoodieBackedTableMetadataWriter.java | 2 +-
...vaInsertOverwriteTableCommitActionExecutor.java | 2 +-
.../client/TestHoodieJavaWriteClientInsert.java | 8 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 10 +-
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 2 +-
.../commit/TestJavaCopyOnWriteActionExecutor.java | 14 +--
.../testutils/HoodieJavaClientTestHarness.java | 11 ++-
.../MultipleSparkJobExecutionStrategy.java | 14 ++-
.../strategy/SingleSparkJobExecutionStrategy.java | 3 +-
.../client/validator/SparkPreCommitValidator.java | 2 +-
.../index/bloom/HoodieFileProbingFunction.java | 3 +-
.../io/storage/HoodieSparkFileReaderFactory.java | 18 ++--
.../io/storage/HoodieSparkFileWriterFactory.java | 11 ++-
.../hudi/io/storage/HoodieSparkIOFactory.java | 15 +--
.../hudi/io/storage/HoodieSparkParquetReader.java | 32 +++----
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 4 +-
.../SparkHoodieBackedTableMetadataWriter.java | 2 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 2 +-
.../SparkDeletePartitionCommitActionExecutor.java | 2 +-
...rkInsertOverwriteTableCommitActionExecutor.java | 3 +-
.../org/apache/hudi/client/TestClientRollback.java | 4 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 2 +-
.../functional/TestExternalPathHandling.java | 3 +-
.../functional/TestHoodieBackedMetadata.java | 40 ++++----
.../functional/TestHoodieBackedTableMetadata.java | 11 ++-
.../TestHoodieClientOnCopyOnWriteStorage.java | 30 +++---
.../TestRemoteFileSystemViewWithMetadataTable.java | 2 +-
.../hudi/index/bloom/TestHoodieBloomIndex.java | 2 +-
.../storage/TestHoodieAvroFileWriterFactory.java | 8 +-
.../commit/TestCopyOnWriteActionExecutor.java | 6 +-
.../table/functional/TestHoodieSparkRollback.java | 7 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 3 +-
.../TestTimelineServerBasedWriteMarkers.java | 3 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 6 +-
.../hudi/testutils/FunctionalTestHarness.java | 3 +-
.../testutils/HoodieSparkClientTestHarness.java | 4 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 7 +-
.../hudi/common/config/HoodieStorageConfig.java | 12 ++-
.../java/org/apache/hudi/common/fs/FSUtils.java | 12 ++-
.../hudi/common/model/HoodieCommitMetadata.java | 16 ++--
.../hudi/common/model/HoodiePartitionMetadata.java | 2 +-
.../hudi/common/table/HoodieTableMetaClient.java | 7 +-
.../hudi/common/table/TableSchemaResolver.java | 4 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 7 +-
.../table/log/block/HoodieAvroDataBlock.java | 4 +-
.../common/table/log/block/HoodieCommandBlock.java | 4 +-
.../common/table/log/block/HoodieCorruptBlock.java | 4 +-
.../common/table/log/block/HoodieDataBlock.java | 8 +-
.../common/table/log/block/HoodieDeleteBlock.java | 4 +-
.../table/log/block/HoodieHFileDataBlock.java | 20 ++--
.../common/table/log/block/HoodieLogBlock.java | 14 +--
.../table/log/block/HoodieParquetDataBlock.java | 12 ++-
.../table/timeline/HoodieActiveTimeline.java | 8 +-
.../common/table/view/FileSystemViewManager.java | 2 +-
.../apache/hudi/common/util/FileFormatUtils.java | 101 ++++++++++----------
.../hudi/common/util/InternalSchemaCache.java | 16 ++--
.../io/FileBasedInternalSchemaStorageManager.java | 15 +--
.../hudi/io/storage/HoodieFileReaderFactory.java | 7 +-
.../hudi/io/storage/HoodieFileWriterFactory.java | 16 ++--
.../apache/hudi/io/storage/HoodieIOFactory.java | 13 ++-
.../io/storage/HoodieNativeAvroHFileReader.java | 13 +--
.../hudi/metadata/AbstractHoodieTableMetadata.java | 7 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 25 +++--
.../metadata/FileSystemBackedTableMetadata.java | 29 +++---
.../hudi/metadata/HoodieBackedTableMetadata.java | 17 +++-
.../metadata/HoodieMetadataFileSystemView.java | 5 +-
.../hudi/metadata/HoodieMetadataMetrics.java | 6 +-
.../hudi/metadata/HoodieMetadataPayload.java | 11 ---
.../apache/hudi/metadata/HoodieTableMetadata.java | 24 +++--
.../hudi/metadata/HoodieTableMetadataUtil.java | 23 ++---
.../main/java/org/apache/hudi/metrics/Metrics.java | 16 ++--
.../apache/hudi/storage/HoodieStorageUtils.java | 13 ++-
.../hudi/common/testutils/HoodieTestUtils.java | 6 ++
.../HoodieBackedTestDelayedTableMetadata.java | 4 +-
.../hudi/sink/bootstrap/BootstrapOperator.java | 6 +-
.../hudi/sink/clustering/ClusteringOperator.java | 6 +-
.../sink/partitioner/profile/WriteProfiles.java | 5 +-
.../java/org/apache/hudi/source/FileIndex.java | 9 +-
.../hudi/source/stats/ColumnStatsIndices.java | 4 +-
.../hudi/table/format/InternalSchemaManager.java | 6 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 5 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 2 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 2 +-
.../common/config/DFSPropertiesConfiguration.java | 3 +-
.../common/table/log/HoodieLogFormatWriter.java | 2 +-
.../org/apache/hudi/common/util/HFileUtils.java | 29 +++---
.../java/org/apache/hudi/common/util/OrcUtils.java | 60 ++++++------
.../org/apache/hudi/common/util/ParquetUtils.java | 103 ++++++++++-----------
.../io/hadoop/HoodieAvroFileReaderFactory.java | 19 ++--
.../io/hadoop/HoodieAvroFileWriterFactory.java | 16 ++--
.../apache/hudi/io/hadoop/HoodieAvroOrcReader.java | 22 ++---
.../hudi/io/hadoop/HoodieAvroParquetReader.java | 19 ++--
.../{storage => hadoop}/HoodieHadoopIOFactory.java | 24 ++---
.../hudi/storage/hadoop/HoodieHadoopStorage.java | 30 ++++--
.../common/functional/TestHoodieLogFormat.java | 4 +-
.../hudi/common/table/TestTableSchemaResolver.java | 7 +-
.../table/log/block/TestHoodieDeleteBlock.java | 2 +-
.../util/TestDFSPropertiesConfiguration.java | 6 +-
.../apache/hudi/common/util/TestParquetUtils.java | 14 +--
.../io/hadoop/TestHoodieAvroFileReaderFactory.java | 13 ++-
.../hadoop/TestHoodieHBaseHFileReaderWriter.java | 17 ++--
.../io/hadoop/TestHoodieHFileReaderWriter.java | 18 ++--
.../io/hadoop/TestHoodieHFileReaderWriterBase.java | 42 ++++-----
.../hudi/io/hadoop/TestHoodieOrcReaderWriter.java | 13 +--
.../hudi/io/hadoop/TestHoodieReaderWriterBase.java | 43 +++++----
.../TestFileSystemBackedTableMetadata.java | 12 +--
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 2 +-
.../hudi/hadoop/HoodieHFileRecordReader.java | 3 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 5 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 5 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 4 +-
.../reader/DFSHoodieDatasetInputReader.java | 4 +-
.../org/apache/hudi/storage/HoodieStorage.java | 25 ++++-
...lkInsertOverwriteTableCommitActionExecutor.java | 4 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 2 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 7 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 18 ++--
.../src/main/scala/org/apache/hudi/Iterators.scala | 2 +-
.../org/apache/hudi/RecordLevelIndexSupport.scala | 4 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 2 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 20 ++--
.../hudi/command/RepairHoodieTableCommand.scala | 16 +++-
.../sql/hudi/streaming/HoodieStreamSource.scala | 3 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 14 +--
.../RepairMigratePartitionMetaProcedure.scala | 3 +-
.../procedures/ShowInvalidParquetProcedure.scala | 6 +-
.../ShowMetadataTableFilesProcedure.scala | 4 +-
.../ShowMetadataTablePartitionsProcedure.scala | 7 +-
.../ShowMetadataTableStatsProcedure.scala | 4 +-
.../ValidateMetadataTableFilesProcedure.scala | 5 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 18 ++--
.../org/apache/hudi/functional/TestBootstrap.java | 14 +--
.../TestSparkConsistentBucketClustering.java | 4 +-
.../row/TestHoodieInternalRowParquetWriter.java | 4 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 1 +
.../apache/hudi/functional/TestCOWDataSource.scala | 8 +-
.../hudi/functional/TestColumnStatsIndex.scala | 8 +-
.../TestMetadataTableWithSparkDataSource.scala | 13 ++-
.../hudi/dml/TestPartialUpdateForMergeInto.scala | 5 +
.../Spark30LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark31LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark32LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark33LegacyHoodieParquetFileFormat.scala | 4 +-
.../Spark34LegacyHoodieParquetFileFormat.scala | 5 +-
.../Spark35LegacyHoodieParquetFileFormat.scala | 4 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 1 +
.../hudi/sync/common/util/ManifestFileWriter.java | 2 +-
.../hudi/utilities/HoodieDataTableValidator.java | 2 +-
.../utilities/HoodieMetadataTableValidator.java | 16 ++--
.../apache/hudi/utilities/HoodieRepairTool.java | 2 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 3 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 7 +-
.../org/apache/hudi/utilities/TableSizeStats.java | 4 +-
.../deltastreamer/HoodieDeltaStreamerMetrics.java | 10 +-
.../ingestion/HoodieIngestionMetrics.java | 12 +--
.../hudi/utilities/perf/TimelineServerPerf.java | 5 +-
.../utilities/streamer/HoodieStreamerMetrics.java | 12 +--
.../apache/hudi/utilities/streamer/StreamSync.java | 7 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 4 +-
.../TestHoodieMetadataTableValidator.java | 2 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 4 +-
215 files changed, 1066 insertions(+), 947 deletions(-)
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java
index 50e71f370db..9c4f70e2025 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java
@@ -110,7 +110,7 @@ public class ArchivedCommitsCommand {
StoragePath archivePath = folder != null && !folder.isEmpty()
? new StoragePath(metaClient.getMetaPath(), folder)
: new StoragePath(metaClient.getArchivePath(), ".commits_.archive*");
- HoodieStorage storage = HoodieStorageUtils.getStorage(metaClient.getBasePathV2(), HoodieCLI.conf);
+ HoodieStorage storage = metaClient.getStorage();
List<StoragePathInfo> pathInfoList = storage.globEntries(archivePath);
List<Comparable[]> allStats = new ArrayList<>();
for (StoragePathInfo pathInfo : pathInfoList) {
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
index b9606fb2f55..7882d6c60cd 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
@@ -187,10 +187,10 @@ public class MetadataCommand {
@ShellMethod(key = "metadata stats", value = "Print stats about the metadata")
public String stats() throws IOException {
- HoodieCLI.getTableMetaClient();
+ HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
- try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(HoodieCLI.conf),
- config, HoodieCLI.basePath)) {
+ try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(
+ new HoodieLocalEngineContext(HoodieCLI.conf), metaClient.getStorage(), config, HoodieCLI.basePath)) {
Map<String, String> stats = metadata.stats();
final List<Comparable[]> rows = new ArrayList<>();
@@ -212,11 +212,11 @@ public class MetadataCommand {
public String listPartitions(
@ShellOption(value = "--sparkMaster", defaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master
) throws IOException {
- HoodieCLI.getTableMetaClient();
+ HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
initJavaSparkContext(Option.of(master));
HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
- try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), config,
- HoodieCLI.basePath)) {
+ try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(
+ new HoodieSparkEngineContext(jsc), metaClient.getStorage(), config, HoodieCLI.basePath)) {
if (!metadata.enabled()) {
return "[ERROR] Metadata Table not enabled/initialized\n\n";
@@ -241,10 +241,10 @@ public class MetadataCommand {
@ShellMethod(key = "metadata list-files", value = "Print a list of all files in a partition from the metadata")
public String listFiles(
@ShellOption(value = {"--partition"}, help = "Name of the partition to list files", defaultValue = "") final String partition) throws IOException {
- HoodieCLI.getTableMetaClient();
+ HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
try (HoodieBackedTableMetadata metaReader = new HoodieBackedTableMetadata(
- new HoodieLocalEngineContext(HoodieCLI.conf), config, HoodieCLI.basePath)) {
+ new HoodieLocalEngineContext(HoodieCLI.conf), metaClient.getStorage(), config, HoodieCLI.basePath)) {
if (!metaReader.enabled()) {
return "[ERROR] Metadata Table not enabled/initialized\n\n";
@@ -278,17 +278,17 @@ public class MetadataCommand {
public String validateFiles(
@ShellOption(value = {"--verbose"}, help = "Print all file details", defaultValue = "false") final boolean verbose)
throws IOException {
- HoodieCLI.getTableMetaClient();
+ HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
HoodieBackedTableMetadata metadataReader = new HoodieBackedTableMetadata(
- new HoodieLocalEngineContext(HoodieCLI.conf), config, HoodieCLI.basePath);
+ new HoodieLocalEngineContext(HoodieCLI.conf), metaClient.getStorage(), config, HoodieCLI.basePath);
if (!metadataReader.enabled()) {
return "[ERROR] Metadata Table not enabled/initialized\n\n";
}
FileSystemBackedTableMetadata fsMetaReader = new FileSystemBackedTableMetadata(new HoodieLocalEngineContext(HoodieCLI.conf),
- HoodieCLI.getTableMetaClient().getTableConfig(), HoodieCLI.conf,
+ HoodieCLI.getTableMetaClient().getTableConfig(), metaClient.getStorage(),
HoodieCLI.basePath, false);
HoodieMetadataConfig fsConfig = HoodieMetadataConfig.newBuilder().enable(false).build();
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
index 2418976c4e4..ab1d49e6648 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
@@ -226,7 +226,7 @@ public class RepairsCommand {
HoodieLocalEngineContext engineContext = new HoodieLocalEngineContext(HoodieCLI.conf);
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
- List<String> partitionPaths = FSUtils.getAllPartitionPaths(engineContext, client.getBasePath(), false, false);
+ List<String> partitionPaths = FSUtils.getAllPartitionPaths(engineContext, client.getStorage(), client.getBasePath(), false, false);
StoragePath basePath = client.getBasePathV2();
String[][] rows = new String[partitionPaths.size()][];
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
index 46ab6bb85ba..c010adc1092 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
@@ -98,7 +98,7 @@ public abstract class BaseHoodieClient implements Serializable, AutoCloseable {
this.heartbeatClient = new HoodieHeartbeatClient(storage, this.basePath,
clientConfig.getHoodieClientHeartbeatIntervalInMs(),
clientConfig.getHoodieClientHeartbeatTolerableMisses());
- this.metrics = new HoodieMetrics(config, context.getStorageConf());
+ this.metrics = new HoodieMetrics(config, storage);
this.txnManager = new TransactionManager(config, storage);
startEmbeddedServerView();
initWrapperFSMetrics();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
index f33acd1e556..5412a60af6a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
@@ -506,7 +506,7 @@ public class HoodieTimelineArchiver<T extends HoodieAvroPayload, I, K, O> {
// If metadata table is enabled, do not archive instants which are more recent than the last compaction on the
// metadata table.
if (table.getMetaClient().getTableConfig().isMetadataTableAvailable()) {
- try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(table.getContext(), config.getMetadataConfig(), config.getBasePath())) {
+ try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(table.getContext(), table.getStorage(), config.getMetadataConfig(), config.getBasePath())) {
Option<String> latestCompactionTime = tableMetadata.getLatestCompactionTime();
if (!latestCompactionTime.isPresent()) {
LOG.info("Not archiving as there is no compaction yet on the metadata table");
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java
index dcdc45932c2..197ef2e6119 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java
@@ -22,8 +22,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.slf4j.Logger;
@@ -100,8 +100,7 @@ public class HeartbeatUtils {
if (config.getFailedWritesCleanPolicy().isLazy() && heartbeatClient.isHeartbeatExpired(instantTime)) {
throw new HoodieException(
"Heartbeat for instant " + instantTime + " has expired, last heartbeat "
- + getLastHeartbeatTime(
- table.getMetaClient().getStorage(), config.getBasePath(), instantTime));
+ + getLastHeartbeatTime(table.getStorage(), config.getBasePath(), instantTime));
}
} catch (IOException io) {
throw new HoodieException("Unable to read heartbeat", io);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
index 4fcb79a588e..b114069b4f7 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
@@ -29,6 +29,7 @@ import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieLockException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
@@ -66,7 +67,7 @@ public class LockManager implements Serializable, AutoCloseable {
Integer.parseInt(HoodieLockConfig.LOCK_ACQUIRE_CLIENT_NUM_RETRIES.defaultValue()));
maxWaitTimeInMs = lockConfiguration.getConfig().getLong(LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY,
Long.parseLong(HoodieLockConfig.LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS.defaultValue()));
- metrics = new HoodieLockMetrics(writeConfig, storageConf);
+ metrics = new HoodieLockMetrics(writeConfig, new HoodieHadoopStorage(fs));
}
public void lock() {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/metrics/HoodieLockMetrics.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/metrics/HoodieLockMetrics.java
index 7a793de5392..a67b6abbc98 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/metrics/HoodieLockMetrics.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/metrics/HoodieLockMetrics.java
@@ -18,16 +18,16 @@
package org.apache.hudi.client.transaction.lock.metrics;
+import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.metrics.Metrics;
+import org.apache.hudi.storage.HoodieStorage;
+
import com.codahale.metrics.Counter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SlidingWindowReservoir;
import com.codahale.metrics.Timer;
-import org.apache.hudi.common.util.HoodieTimer;
-import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.metrics.Metrics;
-import org.apache.hudi.storage.StorageConfiguration;
-
import java.util.concurrent.TimeUnit;
public class HoodieLockMetrics {
@@ -50,12 +50,12 @@ public class HoodieLockMetrics {
private static final Object REGISTRY_LOCK = new Object();
private Metrics metrics;
- public HoodieLockMetrics(HoodieWriteConfig writeConfig, StorageConfiguration<?> storageConf) {
+ public HoodieLockMetrics(HoodieWriteConfig writeConfig, HoodieStorage storage) {
this.isMetricsEnabled = writeConfig.isLockingMetricsEnabled();
this.writeConfig = writeConfig;
if (isMetricsEnabled) {
- metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storageConf);
+ metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storage);
MetricRegistry registry = metrics.getRegistry();
lockAttempts = registry.counter(getMetricsName(LOCK_ACQUIRE_ATTEMPTS_COUNTER_NAME));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
index e7734877198..580fcdd85e0 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
@@ -50,7 +50,7 @@ import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.factory.HoodieAvroKeyGeneratorFactory;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
@@ -179,13 +179,14 @@ public class HoodieIndexUtils {
*
* @param filePath - File to filter keys from
* @param candidateRecordKeys - Candidate keys to filter
+ * @param storage - {@link HoodieStorage} instance
* @return List of candidate keys that are available in the file
*/
public static List<String> filterKeysFromFile(StoragePath filePath, List<String> candidateRecordKeys,
- StorageConfiguration<?> configuration) throws HoodieIndexException {
+ HoodieStorage storage) throws HoodieIndexException {
ValidationUtils.checkArgument(FSUtils.isBaseFile(filePath));
List<String> foundRecordKeys = new ArrayList<>();
- try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(configuration)
+ try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(storage)
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, filePath)) {
// Load all rowKeys from the file, to double-confirm
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java
index 178bc6156e7..25ca3984bde 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java
@@ -57,7 +57,7 @@ public class HoodieGlobalBloomIndex extends HoodieBloomIndex {
List<Pair<String, BloomIndexFileInfo>> loadColumnRangesFromFiles(List<String> partitions, final HoodieEngineContext context,
final HoodieTable hoodieTable) {
HoodieTableMetaClient metaClient = hoodieTable.getMetaClient();
- List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), metaClient.getBasePath());
+ List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(context, metaClient.getStorage(), config.getMetadataConfig(), metaClient.getBasePath());
return super.loadColumnRangesFromFiles(allPartitionPaths, context, hoodieTable);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
index 6023b17ce0d..00ee16b285d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
@@ -187,7 +187,7 @@ public class ConsistentBucketIndexUtils {
* @return true if the metadata is saved successfully
*/
public static boolean saveMetadata(HoodieTable table, HoodieConsistentHashingMetadata metadata, boolean overwrite) {
- HoodieStorage storage = table.getMetaClient().getStorage();
+ HoodieStorage storage = table.getStorage();
StoragePath dir = FSUtils.constructAbsolutePath(
table.getMetaClient().getHashingMetadataPath(), metadata.getPartitionPath());
StoragePath fullPath = new StoragePath(dir, metadata.getFilename());
@@ -211,7 +211,7 @@ public class ConsistentBucketIndexUtils {
* @throws IOException
*/
private static void createCommitMarker(HoodieTable table, Path fileStatus, Path partitionPath) throws IOException {
- HoodieStorage storage = table.getMetaClient().getStorage();
+ HoodieStorage storage = table.getStorage();
StoragePath fullPath = new StoragePath(convertToStoragePath(partitionPath),
getTimestampFromFile(fileStatus.getName()) + HASHING_METADATA_COMMIT_FILE_SUFFIX);
if (storage.exists(fullPath)) {
@@ -240,7 +240,7 @@ public class ConsistentBucketIndexUtils {
if (metaFile == null) {
return Option.empty();
}
- try (InputStream is = table.getMetaClient().getStorage().open(convertToStoragePath(metaFile.getPath()))) {
+ try (InputStream is = table.getStorage().open(convertToStoragePath(metaFile.getPath()))) {
byte[] content = FileIOUtils.readAsByteArray(is);
return Option.of(HoodieConsistentHashingMetadata.fromBytes(content));
} catch (FileNotFoundException e) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/simple/HoodieGlobalSimpleIndex.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/simple/HoodieGlobalSimpleIndex.java
index 5256b036fde..7432d606839 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/simple/HoodieGlobalSimpleIndex.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/simple/HoodieGlobalSimpleIndex.java
@@ -94,7 +94,7 @@ public class HoodieGlobalSimpleIndex extends HoodieSimpleIndex {
private List<Pair<String, HoodieBaseFile>> getAllBaseFilesInTable(
final HoodieEngineContext context, final HoodieTable hoodieTable) {
HoodieTableMetaClient metaClient = hoodieTable.getMetaClient();
- List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), metaClient.getBasePath());
+ List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(context, metaClient.getStorage(), config.getMetadataConfig(), metaClient.getBasePath());
// Obtain the latest data files from all the partitions.
return getLatestBaseFilesForAllPartitions(allPartitionPaths, context, hoodieTable);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
index 07f30c1e3fa..dc61749bf93 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
@@ -105,7 +105,7 @@ public class HoodieCreateHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
createMarkerFile(partitionPath,
FSUtils.makeBaseFileName(this.instantTime, this.writeToken, this.fileId, hoodieTable.getBaseFileExtension()));
this.fileWriter =
- HoodieFileWriterFactory.getFileWriter(instantTime, path, hoodieTable.getStorageConf(), config,
+ HoodieFileWriterFactory.getFileWriter(instantTime, path, hoodieTable.getStorage(), config,
writeSchemaWithMetaFields, this.taskContextSupplier, config.getRecordMerger().getRecordType());
} catch (IOException e) {
throw new HoodieInsertException("Failed to initialize HoodieStorageWriter for path " + path, e);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
index 9db4101cfcb..4d82d661f64 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
@@ -52,9 +52,9 @@ public class HoodieKeyLocationFetchHandle<T, I, K, O> extends HoodieReadHandle<T
private List<HoodieKey> fetchHoodieKeys(HoodieBaseFile baseFile) {
FileFormatUtils fileFormatUtils = FileFormatUtils.getInstance(baseFile.getStoragePath());
if (keyGeneratorOpt.isPresent()) {
- return fileFormatUtils.fetchHoodieKeys(hoodieTable.getStorageConf(), baseFile.getStoragePath(), keyGeneratorOpt);
+ return fileFormatUtils.fetchHoodieKeys(hoodieTable.getStorage(), baseFile.getStoragePath(), keyGeneratorOpt);
} else {
- return fileFormatUtils.fetchHoodieKeys(hoodieTable.getStorageConf(), baseFile.getStoragePath());
+ return fileFormatUtils.fetchHoodieKeys(hoodieTable.getStorage(), baseFile.getStoragePath());
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java
index 664192d454d..6f428329217 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java
@@ -101,7 +101,7 @@ public class HoodieKeyLookupHandle<T, I, K, O> extends HoodieReadHandle<T, I, K,
HoodieBaseFile baseFile = getLatestBaseFile();
List<String> matchingKeys = HoodieIndexUtils.filterKeysFromFile(baseFile.getStoragePath(), candidateRecordKeys,
- hoodieTable.getStorageConf());
+ hoodieTable.getStorage());
LOG.info(
String.format("Total records (%d), bloom filter candidates (%d)/fp(%d), actual matches (%d)", totalKeysChecked,
candidateRecordKeys.size(), candidateRecordKeys.size() - matchingKeys.size(), matchingKeys.size()));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
index 56c183c34e2..e993bd6dc14 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
@@ -197,7 +197,7 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
createMarkerFile(partitionPath, newFilePath.getName());
// Create the writer for writing the new version file
- fileWriter = HoodieFileWriterFactory.getFileWriter(instantTime, newFilePath, hoodieTable.getStorageConf(),
+ fileWriter = HoodieFileWriterFactory.getFileWriter(instantTime, newFilePath, hoodieTable.getStorage(),
config, writeSchemaWithMetaFields, taskContextSupplier, recordMerger.getRecordType());
} catch (IOException io) {
LOG.error("Error in update task at commit " + instantTime, io);
@@ -462,7 +462,7 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
}
long oldNumWrites = 0;
- try (HoodieFileReader reader = HoodieIOFactory.getIOFactory(hoodieTable.getStorageConf())
+ try (HoodieFileReader reader = HoodieIOFactory.getIOFactory(hoodieTable.getStorage())
.getReaderFactory(this.recordMerger.getRecordType())
.getFileReader(config, oldFilePath)) {
oldNumWrites = reader.getTotalRecords();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java
index 71d691ad580..66cf5b3e18d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java
@@ -52,7 +52,7 @@ public abstract class HoodieReadHandle<T, I, K, O> extends HoodieIOHandle<T, I,
@Override
public HoodieStorage getStorage() {
- return hoodieTable.getMetaClient().getStorage();
+ return hoodieTable.getStorage();
}
public Pair<String, String> getPartitionPathFileIDPair() {
@@ -69,12 +69,14 @@ public abstract class HoodieReadHandle<T, I, K, O> extends HoodieIOHandle<T, I,
}
protected HoodieFileReader createNewFileReader() throws IOException {
- return HoodieIOFactory.getIOFactory(hoodieTable.getStorageConf()).getReaderFactory(this.config.getRecordMerger().getRecordType())
+ return HoodieIOFactory.getIOFactory(hoodieTable.getStorage())
+ .getReaderFactory(this.config.getRecordMerger().getRecordType())
.getFileReader(config, getLatestBaseFile().getStoragePath());
}
protected HoodieFileReader createNewFileReader(HoodieBaseFile hoodieBaseFile) throws IOException {
- return HoodieIOFactory.getIOFactory(hoodieTable.getStorageConf()).getReaderFactory(this.config.getRecordMerger().getRecordType())
+ return HoodieIOFactory.getIOFactory(hoodieTable.getStorage())
+ .getReaderFactory(this.config.getRecordMerger().getRecordType())
.getFileReader(config, hoodieBaseFile.getStoragePath());
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
index f51f3d1c279..b3e5c9e30bb 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
@@ -204,7 +204,7 @@ public abstract class HoodieWriteHandle<T, I, K, O> extends HoodieIOHandle<T, I,
@Override
public HoodieStorage getStorage() {
- return hoodieTable.getMetaClient().getStorage();
+ return hoodieTable.getStorage();
}
public HoodieWriteConfig getConfig() {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 46323954a5b..7e6e0e7e3a1 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -66,12 +66,12 @@ import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.exception.TableNotFoundException;
+import org.apache.hudi.io.HoodieMergedReadHandle;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
-import org.apache.hudi.io.HoodieMergedReadHandle;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.hudi.table.HoodieTable;
@@ -191,7 +191,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
}
try {
- this.metadata = new HoodieBackedTableMetadata(engineContext, dataWriteConfig.getMetadataConfig(), dataWriteConfig.getBasePath(), true);
+ this.metadata = new HoodieBackedTableMetadata(engineContext, dataMetaClient.getStorage(), dataWriteConfig.getMetadataConfig(), dataWriteConfig.getBasePath(), true);
this.metadataMetaClient = metadata.getMetadataMetaClient();
} catch (Exception e) {
throw new HoodieException("Could not open MDT for reads", e);
@@ -674,7 +674,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
// List all directories in parallel
engineContext.setJobStatus(this.getClass().getSimpleName(), "Listing " + numDirsToList + " partitions from filesystem");
List<DirectoryInfo> processedDirectories = engineContext.map(pathsToList.subList(0, numDirsToList), path -> {
- HoodieStorage storage = HoodieStorageUtils.getStorage(path, storageConf);
+ HoodieStorage storage = new HoodieHadoopStorage(path, storageConf);
String relativeDirPath = FSUtils.getRelativePartitionPath(storageBasePath, path);
return new DirectoryInfo(relativeDirPath, storage.listDirectEntries(path), initializationTime);
}, numDirsToList);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
index 5edfa7fd4d7..d30f3491a22 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
@@ -25,7 +25,7 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
@@ -94,11 +94,11 @@ public class HoodieMetrics {
private Counter compactionRequestedCounter = null;
private Counter compactionCompletedCounter = null;
- public HoodieMetrics(HoodieWriteConfig config, StorageConfiguration<?> storageConf) {
+ public HoodieMetrics(HoodieWriteConfig config, HoodieStorage storage) {
this.config = config;
this.tableName = config.getTableName();
if (config.isMetricsOn()) {
- metrics = Metrics.getInstance(config.getMetricsConfig(), storageConf);
+ metrics = Metrics.getInstance(config.getMetricsConfig(), storage);
this.rollbackTimerName = getMetricsName("timer", HoodieTimeline.ROLLBACK_ACTION);
this.cleanTimerName = getMetricsName("timer", HoodieTimeline.CLEAN_ACTION);
this.commitTimerName = getMetricsName("timer", HoodieTimeline.COMMIT_ACTION);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
index 009e02277f5..7fa474f7065 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -36,6 +36,8 @@ import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.engine.TaskContextSupplier;
+import org.apache.hudi.common.fs.ConsistencyGuard;
+import org.apache.hudi.common.fs.ConsistencyGuard.FileVisibility;
import org.apache.hudi.common.fs.ConsistencyGuardConfig;
import org.apache.hudi.common.fs.FailSafeConsistencyGuard;
import org.apache.hudi.common.fs.OptimisticConsistencyGuard;
@@ -67,8 +69,6 @@ import org.apache.hudi.exception.HoodieInsertException;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.exception.SchemaCompatibilityException;
-import org.apache.hudi.common.fs.ConsistencyGuard;
-import org.apache.hudi.common.fs.ConsistencyGuard.FileVisibility;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.metadata.HoodieTableMetadata;
@@ -145,7 +145,7 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().fromProperties(config.getMetadataConfig().getProps())
.build();
- this.metadata = HoodieTableMetadata.create(context, metadataConfig, config.getBasePath());
+ this.metadata = HoodieTableMetadata.create(context, metaClient.getStorage(), metadataConfig, config.getBasePath());
this.viewManager = getViewManager();
this.metaClient = metaClient;
@@ -313,6 +313,10 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
return metaClient.getStorageConf();
}
+ public HoodieStorage getStorage() {
+ return metaClient.getStorage();
+ }
+
/**
* Get the view of the file system for this table.
*/
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
index c13a85bfbe6..83d8cbde4a3 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
@@ -69,7 +69,7 @@ public class CleanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K,
public CleanActionExecutor(HoodieEngineContext context, HoodieWriteConfig config, HoodieTable<T, I, K, O> table, String instantTime, boolean skipLocking) {
super(context, config, table, instantTime);
- this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
+ this.txnManager = new TransactionManager(config, table.getStorage());
this.skipLocking = skipLocking;
}
@@ -91,7 +91,7 @@ public class CleanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K,
private static Stream<Pair<String, PartitionCleanStat>> deleteFilesFunc(Iterator<Pair<String, CleanFileInfo>> cleanFileInfo, HoodieTable table) {
Map<String, PartitionCleanStat> partitionCleanStatMap = new HashMap<>();
- FileSystem fs = (FileSystem) table.getMetaClient().getStorage().getFileSystem();
+ FileSystem fs = (FileSystem) table.getStorage().getFileSystem();
cleanFileInfo.forEachRemaining(partitionDelFileTuple -> {
String partitionPath = partitionDelFileTuple.getLeft();
@@ -152,7 +152,7 @@ public class CleanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K,
partitionsToBeDeleted.forEach(entry -> {
try {
if (!isNullOrEmpty(entry)) {
- deleteFileAndGetResult((FileSystem) table.getMetaClient().getStorage().getFileSystem(),
+ deleteFileAndGetResult((FileSystem) table.getStorage().getFileSystem(),
table.getMetaClient().getBasePath() + "/" + entry);
}
} catch (IOException e) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
index b881a0f060e..0eb746c1083 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
@@ -280,7 +280,7 @@ public class CleanPlanner<T, I, K, O> implements Serializable {
*/
private List<String> getPartitionPathsForFullCleaning() {
// Go to brute force mode of scanning all partitions
- return FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), config.getBasePath());
+ return FSUtils.getAllPartitionPaths(context, hoodieTable.getStorage(), config.getMetadataConfig(), config.getBasePath());
}
/**
@@ -468,7 +468,7 @@ public class CleanPlanner<T, I, K, O> implements Serializable {
HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(hoodieTable.getMetaClient(), hoodieTable.getActiveTimeline());
StoragePath fullPartitionPath = new StoragePath(hoodieTable.getMetaClient().getBasePathV2(), partitionPath);
fsView.addFilesToView(partitionPath, FSUtils.getAllDataFilesInPartition(
- hoodieTable.getMetaClient().getStorage(), fullPartitionPath));
+ hoodieTable.getStorage(), fullPartitionPath));
// use #getAllFileGroups(partitionPath) instead of #getAllFileGroups() to exclude the replaced file groups.
return fsView.getAllFileGroups(partitionPath).findAny().isPresent();
} catch (Exception ex) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
index c22c8b24eee..59789ab3f07 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
@@ -135,7 +135,8 @@ public abstract class PartitionAwareClusteringPlanStrategy<T,I,K,O> extends Clus
if (StringUtils.isNullOrEmpty(partitionSelected)) {
// get matched partitions if set
- partitionPaths = getRegexPatternMatchedPartitions(config, FSUtils.getAllPartitionPaths(getEngineContext(), config.getMetadataConfig(), metaClient.getBasePath()));
+ partitionPaths = getRegexPatternMatchedPartitions(config, FSUtils.getAllPartitionPaths(
+ getEngineContext(), metaClient.getStorage(), config.getMetadataConfig(), metaClient.getBasePath()));
// filter the partition paths if needed to reduce list status
} else {
partitionPaths = Arrays.asList(partitionSelected.split(","));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
index aaad57f6079..2d4a24ce3a3 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
@@ -94,7 +94,7 @@ public abstract class BaseCommitActionExecutor<T, I, K, O, R>
this.taskContextSupplier = context.getTaskContextSupplier();
// TODO : Remove this once we refactor and move out autoCommit method from here, since the TxnManager is held in {@link BaseHoodieWriteClient}.
this.txnManagerOption = config.shouldAutoCommit()
- ? Option.of(new TransactionManager(config, table.getMetaClient().getStorage())) : Option.empty();
+ ? Option.of(new TransactionManager(config, table.getStorage())) : Option.empty();
if (this.txnManagerOption.isPresent() && this.txnManagerOption.get().isLockRequired()) {
// these txn metadata are only needed for auto commit when optimistic concurrent control is also enabled
this.lastCompletedTxn = TransactionUtils.getLastCompletedTxnInstantAndMetadata(table.getMetaClient());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java
index cfd9ff606dd..3f661e3b10b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java
@@ -38,7 +38,7 @@ import org.apache.hudi.internal.schema.utils.SerDeHelper;
import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.util.ExecutorFactory;
@@ -78,9 +78,9 @@ public class HoodieMergeHelper<T> extends BaseMergeHelper {
HoodieWriteConfig writeConfig = table.getConfig();
HoodieBaseFile baseFile = mergeHandle.baseFileForMerge();
- StorageConfiguration<?> storageConf = table.getStorageConf().newInstance();
HoodieRecord.HoodieRecordType recordType = table.getConfig().getRecordMerger().getRecordType();
- HoodieFileReader baseFileReader = HoodieIOFactory.getIOFactory(storageConf)
+ HoodieFileReader baseFileReader = HoodieIOFactory.getIOFactory(
+ table.getStorage().newInstance(mergeHandle.getOldFilePath(), table.getStorageConf().newInstance()))
.getReaderFactory(recordType)
.getFileReader(writeConfig, mergeHandle.getOldFilePath());
HoodieFileReader bootstrapFileReader = null;
@@ -111,10 +111,11 @@ public class HoodieMergeHelper<T> extends BaseMergeHelper {
Schema recordSchema;
if (baseFile.getBootstrapBaseFile().isPresent()) {
StoragePath bootstrapFilePath = baseFile.getBootstrapBaseFile().get().getStoragePath();
- StorageConfiguration<?> bootstrapFileConfig = table.getStorageConf().newInstance();
- bootstrapFileReader = HoodieIOFactory.getIOFactory(bootstrapFileConfig).getReaderFactory(recordType).newBootstrapFileReader(
+ HoodieStorage storage = table.getStorage().newInstance(
+ bootstrapFilePath, table.getStorageConf().newInstance());
+ bootstrapFileReader = HoodieIOFactory.getIOFactory(storage).getReaderFactory(recordType).newBootstrapFileReader(
baseFileReader,
- HoodieIOFactory.getIOFactory(bootstrapFileConfig).getReaderFactory(recordType)
+ HoodieIOFactory.getIOFactory(storage).getReaderFactory(recordType)
.getFileReader(writeConfig, bootstrapFilePath),
mergeHandle.getPartitionFields(),
mergeHandle.getPartitionValues());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/RunCompactionActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/RunCompactionActionExecutor.java
index 55e8ce7d23f..f8ce1fcdc2a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/RunCompactionActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/RunCompactionActionExecutor.java
@@ -73,7 +73,7 @@ public class RunCompactionActionExecutor<T> extends
this.operationType = operationType;
checkArgument(operationType == WriteOperationType.COMPACT || operationType == WriteOperationType.LOG_COMPACT,
"Only COMPACT and LOG_COMPACT is supported");
- metrics = new HoodieMetrics(config, context.getStorageConf());
+ metrics = new HoodieMetrics(config, table.getStorage());
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
index 2d528227797..3c84bfb9d20 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
@@ -80,7 +80,7 @@ public abstract class BaseHoodieCompactionPlanGenerator<T extends HoodieRecordPa
// TODO : check if maxMemory is not greater than JVM or executor memory
// TODO - rollback any compactions in flight
HoodieTableMetaClient metaClient = hoodieTable.getMetaClient();
- List<String> partitionPaths = FSUtils.getAllPartitionPaths(engineContext, writeConfig.getMetadataConfig(), metaClient.getBasePath());
+ List<String> partitionPaths = FSUtils.getAllPartitionPaths(engineContext, metaClient.getStorage(), writeConfig.getMetadataConfig(), metaClient.getBasePath());
// filter the partition paths if needed to reduce list status
partitionPaths = filterPartitionPathsByStrategy(writeConfig, partitionPaths);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RecordBasedIndexingCatchupTask.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RecordBasedIndexingCatchupTask.java
index 53f357c03f7..04e51f9774e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RecordBasedIndexingCatchupTask.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RecordBasedIndexingCatchupTask.java
@@ -66,7 +66,8 @@ public class RecordBasedIndexingCatchupTask extends AbstractIndexingCatchupTask
private HoodieData<HoodieRecord> readRecordKeysFromFileSlices(HoodieInstant instant) throws IOException {
HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).build();
- HoodieTableMetadata metadata = HoodieTableMetadata.create(engineContext, metadataConfig, metaClient.getBasePathV2().toString(), false);
+ HoodieTableMetadata metadata = HoodieTableMetadata.create(
+ engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePathV2().toString(), false);
HoodieMetadataFileSystemView fsView = new HoodieMetadataFileSystemView(metaClient, metaClient.getActiveTimeline().filter(i -> i.equals(instant)), metadata);
// Collect the list of latest file slices present in each partition
List<String> partitions = metadata.getAllPartitionPaths();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
index 5ad4e5e9f39..106a41a6823 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
@@ -97,9 +97,9 @@ public class RunIndexActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I,
public RunIndexActionExecutor(HoodieEngineContext context, HoodieWriteConfig config, HoodieTable<T, I, K, O> table, String instantTime) {
super(context, config, table, instantTime);
- this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
+ this.txnManager = new TransactionManager(config, table.getStorage());
if (config.getMetadataConfig().isMetricsEnabled()) {
- this.metrics = Option.of(new HoodieMetadataMetrics(config.getMetricsConfig(), context.getStorageConf()));
+ this.metrics = Option.of(new HoodieMetadataMetrics(config.getMetricsConfig(), table.getStorage()));
} else {
this.metrics = Option.empty();
}
@@ -212,7 +212,7 @@ public class RunIndexActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I,
});
table.getMetaClient().getTableConfig().setValue(TABLE_METADATA_PARTITIONS_INFLIGHT.key(), String.join(",", inflightPartitions));
table.getMetaClient().getTableConfig().setValue(TABLE_METADATA_PARTITIONS.key(), String.join(",", completedPartitions));
- HoodieTableConfig.update(table.getMetaClient().getStorage(),
+ HoodieTableConfig.update(table.getStorage(),
table.getMetaClient().getMetaPath(), table.getMetaClient().getTableConfig().getProps());
// delete metadata partition
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java
index b827e53dd0b..548a7b18fce 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java
@@ -74,7 +74,7 @@ public class ScheduleIndexActionExecutor<T, I, K, O> extends BaseActionExecutor<
List<MetadataPartitionType> partitionIndexTypes) {
super(context, config, table, instantTime);
this.partitionIndexTypes = partitionIndexTypes;
- this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
+ this.txnManager = new TransactionManager(config, table.getStorage());
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
index ad00fe052df..30a02f2b458 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
@@ -62,7 +62,7 @@ public abstract class BaseRestoreActionExecutor<T, I, K, O> extends BaseActionEx
String savepointToRestoreTimestamp) {
super(context, config, table, instantTime);
this.savepointToRestoreTimestamp = savepointToRestoreTimestamp;
- this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
+ this.txnManager = new TransactionManager(config, table.getStorage());
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
index 906bb64ac2e..868b4e2a718 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
@@ -89,7 +89,7 @@ public abstract class BaseRollbackActionExecutor<T, I, K, O> extends BaseActionE
this.deleteInstants = deleteInstants;
this.skipTimelinePublish = skipTimelinePublish;
this.skipLocking = skipLocking;
- this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
+ this.txnManager = new TransactionManager(config, table.getStorage());
}
/**
@@ -177,7 +177,7 @@ public abstract class BaseRollbackActionExecutor<T, I, K, O> extends BaseActionE
&& !commitTimeline.findInstantsAfter(instantTimeToRollback, Integer.MAX_VALUE).empty()) {
// check if remnants are from a previous LAZY rollback config, if yes, let out of order rollback continue
try {
- if (!HoodieHeartbeatClient.heartbeatExists(table.getMetaClient().getStorage(),
+ if (!HoodieHeartbeatClient.heartbeatExists(table.getStorage(),
config.getBasePath(), instantTimeToRollback)) {
throw new HoodieRollbackException(
"Found commits after time :" + instantTimeToRollback + ", please rollback greater commits first");
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
index 39f6d8c3ca1..ea949e23326 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
@@ -94,7 +94,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
try {
HoodieTableMetaClient metaClient = table.getMetaClient();
List<String> partitionPaths =
- FSUtils.getAllPartitionPaths(context, table.getMetaClient().getBasePath(), false, false);
+ FSUtils.getAllPartitionPaths(context, table.getStorage(), table.getMetaClient().getBasePath(), false, false);
int numPartitions = Math.max(Math.min(partitionPaths.size(), config.getRollbackParallelism()), 1);
context.setJobStatus(this.getClass().getSimpleName(), "Creating Listing Rollback Plan: " + config.getTableName());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
index 1e0330a4def..b1ed3f2417a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
@@ -123,7 +123,8 @@ public class SavepointActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I
return latestFiles;
}));
} else {
- List<String> partitions = FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), table.getMetaClient().getBasePath());
+ List<String> partitions = FSUtils.getAllPartitionPaths(
+ context, table.getStorage(), config.getMetadataConfig(), table.getMetaClient().getBasePath());
latestFilesMap = context.mapToPair(partitions, partitionPath -> {
// Scan all partitions files with this commit time
LOG.info("Collecting latest files in partition path " + partitionPath);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java
index 77498e08750..8b89294e281 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java
@@ -78,7 +78,7 @@ public class DirectWriteMarkers extends WriteMarkers {
}
public DirectWriteMarkers(HoodieTable table, String instantTime) {
- this(table.getMetaClient().getStorage(),
+ this(table.getStorage(),
table.getMetaClient().getBasePath(),
table.getMetaClient().getMarkerFolderPath(instantTime),
instantTime);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java
index af1819f4cda..e3caacf402e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java
@@ -63,7 +63,7 @@ public class MarkerBasedRollbackUtils {
public static List<String> getAllMarkerPaths(HoodieTable table, HoodieEngineContext context,
String instant, int parallelism) throws IOException {
String markerDir = table.getMetaClient().getMarkerFolderPath(instant);
- HoodieStorage storage = table.getMetaClient().getStorage();
+ HoodieStorage storage = table.getStorage();
Option<MarkerType> markerTypeOption = readMarkerType(storage, markerDir);
// If there is no marker type file "MARKERS.type", first assume "DIRECT" markers are used.
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
index bff523f7f21..8a2cea2006c 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
@@ -23,16 +23,16 @@ import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ParquetUtils;
import org.apache.hudi.io.hadoop.HoodieAvroParquetWriter;
import org.apache.hudi.io.storage.HoodieParquetConfig;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
@@ -47,7 +47,6 @@ import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
-import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -57,7 +56,7 @@ public class TestHoodieAvroParquetWriter {
@Test
public void testProperWriting() throws IOException {
- StorageConfiguration<Configuration> storageConf = getDefaultStorageConf();
+ HoodieStorage storage = HoodieTestUtils.getStorage(tmpDir.toString());
HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(0xDEED);
List<GenericRecord> records = dataGen.generateGenericRecords(10);
@@ -71,7 +70,7 @@ public class TestHoodieAvroParquetWriter {
HoodieParquetConfig<HoodieAvroWriteSupport> parquetConfig =
new HoodieParquetConfig(writeSupport, CompressionCodecName.GZIP, ParquetWriter.DEFAULT_BLOCK_SIZE,
- ParquetWriter.DEFAULT_PAGE_SIZE, 1024 * 1024 * 1024, storageConf, 0.1, true);
+ ParquetWriter.DEFAULT_PAGE_SIZE, 1024 * 1024 * 1024, storage.getConf(), 0.1, true);
StoragePath filePath = new StoragePath(tmpDir.resolve("test.parquet").toAbsolutePath().toString());
@@ -85,7 +84,7 @@ public class TestHoodieAvroParquetWriter {
ParquetUtils utils = new ParquetUtils();
// Step 1: Make sure records are written appropriately
- List<GenericRecord> readRecords = utils.readAvroRecords(storageConf, filePath);
+ List<GenericRecord> readRecords = utils.readAvroRecords(storage, filePath);
assertEquals(toJson(records), toJson(readRecords));
@@ -95,8 +94,7 @@ public class TestHoodieAvroParquetWriter {
String minKey = recordKeys.stream().min(Comparator.naturalOrder()).get();
String maxKey = recordKeys.stream().max(Comparator.naturalOrder()).get();
- FileMetaData parquetMetadata = ParquetUtils.readMetadata(
- storageConf, filePath).getFileMetaData();
+ FileMetaData parquetMetadata = ParquetUtils.readMetadata(storage, filePath).getFileMetaData();
Map<String, String> extraMetadata = parquetMetadata.getKeyValueMetaData();
@@ -105,7 +103,7 @@ public class TestHoodieAvroParquetWriter {
assertEquals(extraMetadata.get(HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE), BloomFilterTypeCode.DYNAMIC_V0.name());
// Step 3: Make sure Bloom Filter contains all the record keys
- BloomFilter bloomFilter = utils.readBloomFilterFromMetadata(storageConf, filePath);
+ BloomFilter bloomFilter = utils.readBloomFilterFromMetadata(storage, filePath);
recordKeys.forEach(recordKey -> {
assertTrue(bloomFilter.mightContain(recordKey));
});
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestCommitMetadataUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestCommitMetadataUtils.java
index d8cd9d22050..37bf950569c 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestCommitMetadataUtils.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestCommitMetadataUtils.java
@@ -92,7 +92,7 @@ public class TestCommitMetadataUtils extends HoodieCommonTestHarness {
when(table.getMetaClient()).thenReturn(metaClient);
Mockito.when(table.getConfig()).thenReturn(writeConfig);
when(metaClient.getTableType()).thenReturn(HoodieTableType.MERGE_ON_READ);
- when(metaClient.getStorage()).thenReturn(new HoodieHadoopStorage(fileSystem));
+ when(metaClient.getStorage()).thenReturn(new HoodieHadoopStorage(basePath, getDefaultStorageConf()));
when(metaClient.getBasePath()).thenReturn(basePath);
when(metaClient.getMarkerFolderPath(any())).thenReturn(basePath + ".hoodie/.temp");
when(table.getContext()).thenReturn(context);
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java
index 4e938ef1cef..c156997441b 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java
@@ -21,7 +21,6 @@ package org.apache.hudi.metrics;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -42,7 +41,6 @@ public class TestHoodieConsoleMetrics {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -52,7 +50,7 @@ public class TestHoodieConsoleMetrics {
when(writeConfig.isMetricsOn()).thenReturn(true);
when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.CONSOLE);
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieGraphiteMetrics.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieGraphiteMetrics.java
index cf488405660..250bffa25aa 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieGraphiteMetrics.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieGraphiteMetrics.java
@@ -22,7 +22,6 @@ import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.NetworkTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
@@ -45,7 +44,6 @@ public class TestHoodieGraphiteMetrics {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -63,7 +61,7 @@ public class TestHoodieGraphiteMetrics {
when(metricsConfig.getGraphiteServerPort()).thenReturn(NetworkTestUtils.nextFreePort());
when(metricsConfig.getGraphiteReportPeriodSeconds()).thenReturn(30);
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
metrics.registerGauge("graphite_metric", 123L);
assertEquals("123", metrics.getRegistry().getGauges()
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
index 9daebd08661..20542cc4cc2 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
@@ -22,7 +22,6 @@ import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.NetworkTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -46,7 +45,6 @@ public class TestHoodieJmxMetrics {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -58,7 +56,7 @@ public class TestHoodieJmxMetrics {
when(metricsConfig.getJmxHost()).thenReturn("localhost");
when(metricsConfig.getJmxPort()).thenReturn(String.valueOf(NetworkTestUtils.nextFreePort()));
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
index 39cd0dc444f..83387a64a17 100755
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
@@ -25,7 +25,6 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.index.HoodieIndex;
-import org.apache.hudi.storage.StorageConfiguration;
import com.codahale.metrics.Timer;
import org.junit.jupiter.api.AfterEach;
@@ -51,7 +50,6 @@ public class TestHoodieMetrics {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -61,7 +59,7 @@ public class TestHoodieMetrics {
when(writeConfig.isMetricsOn()).thenReturn(true);
when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY);
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java
index 9a7b82b4485..8d57c909cd5 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java
@@ -18,13 +18,13 @@
package org.apache.hudi.metrics.datadog;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
-import org.apache.hudi.storage.StorageConfiguration;
import com.codahale.metrics.MetricRegistry;
import org.junit.jupiter.api.AfterEach;
@@ -48,8 +48,6 @@ public class TestDatadogMetricsReporter {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- @Mock
- StorageConfiguration storageConf;
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -73,7 +71,7 @@ public class TestDatadogMetricsReporter {
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
Throwable t = assertThrows(IllegalStateException.class, () -> {
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
});
assertEquals("Datadog cannot be initialized: API key is null or empty.", t.getMessage());
@@ -89,7 +87,7 @@ public class TestDatadogMetricsReporter {
when(metricsConfig.getDatadogMetricPrefix()).thenReturn("");
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
Throwable t = assertThrows(IllegalStateException.class, () -> {
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
});
assertEquals("Datadog cannot be initialized: Metric prefix is null or empty.", t.getMessage());
@@ -111,7 +109,7 @@ public class TestDatadogMetricsReporter {
when(metricsConfig.getMetricReporterMetricsNamePrefix()).thenReturn("");
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
assertDoesNotThrow(() -> {
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
});
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/m3/TestM3Metrics.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/m3/TestM3Metrics.java
index 954619f6174..76917c22f99 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/m3/TestM3Metrics.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/m3/TestM3Metrics.java
@@ -18,18 +18,13 @@
package org.apache.hudi.metrics.m3;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.when;
-
-import java.util.UUID;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.NetworkTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -37,6 +32,12 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.when;
+
@ExtendWith(MockitoExtension.class)
public class TestM3Metrics {
@@ -44,8 +45,6 @@ public class TestM3Metrics {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- @Mock
- StorageConfiguration storageConf;
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -66,7 +65,7 @@ public class TestM3Metrics {
when(metricsConfig.getM3Service()).thenReturn("hoodie");
when(metricsConfig.getM3Tags()).thenReturn("tag1=value1,tag2=value2");
when(metricsConfig.getMetricReporterMetricsNamePrefix()).thenReturn("");
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
metrics.registerGauge("metric1", 123L);
assertEquals("123", metrics.getRegistry().getGauges().get("metric1").getValue().toString());
@@ -84,7 +83,7 @@ public class TestM3Metrics {
when(metricsConfig.getM3Service()).thenReturn("hoodie");
when(metricsConfig.getM3Tags()).thenReturn("");
when(metricsConfig.getMetricReporterMetricsNamePrefix()).thenReturn("");
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
metrics.registerGauge("metric1", 123L);
assertEquals("123", metrics.getRegistry().getGauges().get("metric1").getValue().toString());
@@ -98,7 +97,7 @@ public class TestM3Metrics {
when(writeConfig.isMetricsOn()).thenReturn(true);
when(metricsConfig.getMetricReporterMetricsNamePrefix()).thenReturn("");
assertThrows(RuntimeException.class, () -> {
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
});
}
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java
index d95614a577a..e1aee8793f4 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java
@@ -24,7 +24,6 @@ import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
@@ -44,7 +43,6 @@ public class TestPrometheusReporter {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -63,8 +61,8 @@ public class TestPrometheusReporter {
when(metricsConfig.getPrometheusPort()).thenReturn(9090);
when(metricsConfig.getBasePath()).thenReturn("s3://test" + UUID.randomUUID());
assertDoesNotThrow(() -> {
- new HoodieMetrics(writeConfig, storageConf);
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
});
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java
index c2c7695932d..20f8fe5e105 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java
@@ -26,7 +26,6 @@ import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.metrics.MetricUtils;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
-import org.apache.hudi.storage.StorageConfiguration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
@@ -62,7 +61,6 @@ public class TestPushGateWayReporter {
HoodieWriteConfig writeConfig;
@Mock
HoodieMetricsConfig metricsConfig;
- StorageConfiguration storageConf = HoodieTestUtils.getDefaultStorageConf();
HoodieMetrics hoodieMetrics;
Metrics metrics;
@@ -81,7 +79,7 @@ public class TestPushGateWayReporter {
configureDefaultReporter();
assertDoesNotThrow(() -> {
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
});
@@ -106,7 +104,7 @@ public class TestPushGateWayReporter {
when(metricsConfig.getMetricReporterFileBasedConfigs()).thenReturn(propPrometheusPath + "," + propDatadogPath);
}
- hoodieMetrics = new HoodieMetrics(writeConfig, storageConf);
+ hoodieMetrics = new HoodieMetrics(writeConfig, HoodieTestUtils.getDefaultStorage());
metrics = hoodieMetrics.getMetrics();
Map<String, Long> metricsMap = new HashMap<>();
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
index 1b1bb6bcfaa..b03441ae449 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
@@ -27,12 +27,13 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieValidationException;
import org.apache.hudi.hadoop.config.HoodieRealtimeConfig;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieIOFactory;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
@@ -145,9 +146,9 @@ public class GenericRecordValidationTestUtils {
public static Stream<GenericRecord> readHFile(Configuration conf, String[] paths) {
List<GenericRecord> valuesAsList = new LinkedList<>();
for (String path : paths) {
- StorageConfiguration storageConf = HadoopFSUtils.getStorageConf(conf);
+ HoodieStorage storage = new HoodieHadoopStorage(path, conf);
try (HoodieAvroHFileReaderImplBase reader = (HoodieAvroHFileReaderImplBase)
- HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
+ HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, new StoragePath(path), HoodieFileFormat.HFILE)) {
valuesAsList.addAll(HoodieAvroHFileReaderImplBase.readAllRecords(reader)
.stream().map(e -> (GenericRecord) e).collect(Collectors.toList()));
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java
index 30dc4b842be..b31d4470d43 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java
@@ -477,7 +477,7 @@ public class HoodieFlinkWriteClient<T> extends
case INSERT_OVERWRITE_TABLE:
Map<String, List<String>> partitionToExistingFileIds = new HashMap<>();
List<String> partitionPaths =
- FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), table.getMetaClient().getBasePath());
+ FSUtils.getAllPartitionPaths(context, table.getStorage(), config.getMetadataConfig(), table.getMetaClient().getBasePath());
if (partitionPaths != null && partitionPaths.size() > 0) {
context.setJobStatus(this.getClass().getSimpleName(), "Getting ExistingFileIds of all partitions: " + config.getTableName());
partitionToExistingFileIds = partitionPaths.stream().parallel()
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
index 5915a3eda36..156e3aa1339 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
@@ -34,8 +34,8 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieInsertException;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
@@ -92,7 +92,7 @@ public class HoodieRowDataCreateHandle implements Serializable {
this.newRecordLocation = new HoodieRecordLocation(instantTime, fileId);
this.preserveHoodieMetadata = preserveHoodieMetadata;
this.currTimer = HoodieTimer.start();
- this.storage = table.getMetaClient().getStorage();
+ this.storage = table.getStorage();
this.path = makeNewPath(partitionPath);
this.writeStatus = new WriteStatus(table.shouldTrackSuccessRecords(),
@@ -176,8 +176,7 @@ public class HoodieRowDataCreateHandle implements Serializable {
stat.setFileId(fileId);
StoragePath storagePath = convertToStoragePath(path);
stat.setPath(new StoragePath(writeConfig.getBasePath()), storagePath);
- long fileSizeInBytes = FSUtils.getFileSize(
- table.getMetaClient().getStorage(), storagePath);
+ long fileSizeInBytes = FSUtils.getFileSize(table.getStorage(), storagePath);
stat.setTotalWriteBytes(fileSizeInBytes);
stat.setFileSizeInBytes(fileSizeInBytes);
stat.setTotalWriteErrors(writeStatus.getTotalErrorRecords());
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
index 77f1439c982..26bf7140baa 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
@@ -88,7 +88,7 @@ public class FlinkHoodieBackedTableMetadataWriter extends HoodieBackedTableMetad
protected void initRegistry() {
if (metadataWriteConfig.isMetricsOn()) {
// should support executor metrics
- this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), storageConf));
+ this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage()));
} else {
this.metrics = Option.empty();
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java
index 54c079b5166..90d12a2c496 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java
@@ -80,7 +80,7 @@ public class FlinkDeletePartitionCommitActionExecutor<T extends HoodieRecordPayl
// created requested
HoodieInstant dropPartitionsInstant =
new HoodieInstant(REQUESTED, REPLACE_COMMIT_ACTION, instantTime);
- if (!table.getMetaClient().getStorage().exists(new StoragePath(
+ if (!table.getStorage().exists(new StoragePath(
table.getMetaClient().getMetaPath(), dropPartitionsInstant.getFileName()))) {
HoodieRequestedReplaceMetadata requestedReplaceMetadata =
HoodieRequestedReplaceMetadata.newBuilder()
diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java
index 1046a84a52e..dd89c6ae9d6 100644
--- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java
+++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java
@@ -246,9 +246,8 @@ public class TestFlinkHoodieBloomIndex extends HoodieFlinkClientTestHarness {
List<String> uuids = asList(record1.getRecordKey(), record2.getRecordKey(), record3.getRecordKey(), record4.getRecordKey());
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).build();
- HoodieFlinkTable table = HoodieFlinkTable.create(config, context, metaClient);
List<String> results = HoodieIndexUtils.filterKeysFromFile(
- new StoragePath(java.nio.file.Paths.get(basePath, partition, filename).toString()), uuids, storageConf);
+ new StoragePath(java.nio.file.Paths.get(basePath, partition, filename).toString()), uuids, metaClient.getStorage());
assertEquals(results.size(), 2);
assertTrue(results.get(0).equals("1eb5b87a-1feh-4edd-87b4-6ec96dc405a0")
|| results.get(1).equals("1eb5b87a-1feh-4edd-87b4-6ec96dc405a0"));
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
index 6dd0fc09d72..5f0a3a2c64b 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
@@ -175,7 +175,7 @@ public abstract class JavaExecutionStrategy<T>
try {
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(config.getSchema()));
scanner = HoodieMergedLogRecordScanner.newBuilder()
- .withStorage(table.getMetaClient().getStorage())
+ .withStorage(table.getStorage())
.withBasePath(table.getMetaClient().getBasePath())
.withLogFilePaths(clusteringOp.getDeltaFilePaths())
.withReaderSchema(readerSchema)
@@ -192,7 +192,7 @@ public abstract class JavaExecutionStrategy<T>
baseFileReader = StringUtils.isNullOrEmpty(clusteringOp.getDataFilePath())
? Option.empty()
- : Option.of(HoodieIOFactory.getIOFactory(table.getStorageConf()).getReaderFactory(recordType)
+ : Option.of(HoodieIOFactory.getIOFactory(table.getStorage()).getReaderFactory(recordType)
.getFileReader(config, new StoragePath(clusteringOp.getDataFilePath())));
HoodieTableConfig tableConfig = table.getMetaClient().getTableConfig();
Iterator<HoodieRecord<T>> fileSliceReader = new HoodieFileSliceReader(baseFileReader, scanner, readerSchema, tableConfig.getPreCombineField(), writeConfig.getRecordMerger(),
@@ -221,7 +221,8 @@ public abstract class JavaExecutionStrategy<T>
private List<HoodieRecord<T>> readRecordsForGroupBaseFiles(List<ClusteringOperation> clusteringOps) {
List<HoodieRecord<T>> records = new ArrayList<>();
clusteringOps.forEach(clusteringOp -> {
- try (HoodieFileReader baseFileReader = HoodieIOFactory.getIOFactory(getHoodieTable().getStorageConf()).getReaderFactory(recordType)
+ try (HoodieFileReader baseFileReader = HoodieIOFactory.getIOFactory(getHoodieTable().getStorage())
+ .getReaderFactory(recordType)
.getFileReader(getHoodieTable().getConfig(), new StoragePath(clusteringOp.getDataFilePath()))) {
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(getWriteConfig().getSchema()));
Iterator<HoodieRecord> recordIterator = baseFileReader.getRecordIterator(readerSchema);
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
index 1c362c35e85..180c2b70fbd 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
@@ -73,7 +73,7 @@ public class JavaHoodieBackedTableMetadataWriter extends HoodieBackedTableMetada
@Override
protected void initRegistry() {
if (metadataWriteConfig.isMetricsOn()) {
- this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), storageConf));
+ this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage()));
} else {
this.metrics = Option.empty();
}
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertOverwriteTableCommitActionExecutor.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertOverwriteTableCommitActionExecutor.java
index 3b4131e8235..76f8f9752ca 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertOverwriteTableCommitActionExecutor.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertOverwriteTableCommitActionExecutor.java
@@ -51,7 +51,7 @@ public class JavaInsertOverwriteTableCommitActionExecutor<T>
protected Map<String, List<String>> getPartitionToReplacedFileIds(HoodieWriteMetadata<List<WriteStatus>> writeResult) {
Map<String, List<String>> partitionToExistingFileIds = new HashMap<>();
List<String> partitionPaths = FSUtils.getAllPartitionPaths(context,
- table.getMetaClient().getBasePath(), config.isMetadataTableEnabled(), config.shouldAssumeDatePartitioning());
+ table.getStorage(), table.getMetaClient().getBasePath(), config.isMetadataTableEnabled(), config.shouldAssumeDatePartitioning());
if (partitionPaths != null && partitionPaths.size() > 0) {
partitionToExistingFileIds = context.mapToPair(partitionPaths,
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
index 718203561c7..53d069736e7 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
@@ -172,7 +172,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storageConf, new StoragePath(filePath.toUri()));
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storage, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records1) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
@@ -204,7 +204,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
records1.addAll(records2);
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
int index = 0;
for (GenericRecord record : fileRecords) {
assertEquals(records1.get(index).getRecordKey(), record.get("_row_key").toString());
@@ -239,7 +239,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storageConf, new StoragePath(filePath.toUri()));
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storage, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records1) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
@@ -260,7 +260,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
records1.addAll(records2);
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
assertEquals(fileRecords.size(), mergeAllowDuplicateOnInsertsEnable ? records1.size() : records2.size());
int index = 0;
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
index d697c192221..5d58059b573 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
@@ -544,7 +544,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
List<FileSlice> fileSlices = table.getSliceView().getLatestFileSlices("files").collect(Collectors.toList());
HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
- HoodieIOFactory.getIOFactory(context.getStorageConf()).getReaderFactory(HoodieRecordType.AVRO)
+ HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(writeConfig, new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
records.forEach(entry -> {
@@ -971,7 +971,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
final HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
- HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecordType.AVRO)
+ HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(table.getConfig(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
records.forEach(entry -> {
@@ -2376,7 +2376,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
assertNoWriteErrors(writeStatuses);
validateMetadata(client);
- Metrics metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storageConf);
+ Metrics metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storage);
assertTrue(metrics.getRegistry().getGauges().containsKey(HoodieMetadataMetrics.INITIALIZE_STR + ".count"));
assertTrue(metrics.getRegistry().getGauges().containsKey(HoodieMetadataMetrics.INITIALIZE_STR + ".totalDuration"));
assertTrue((Long) metrics.getRegistry().getGauges().get(HoodieMetadataMetrics.INITIALIZE_STR + ".count").getValue() >= 1L);
@@ -2684,7 +2684,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// Partitions should match
FileSystemBackedTableMetadata fsBackedTableMetadata = new FileSystemBackedTableMetadata(engineContext, metaClient.getTableConfig(),
- storageConf.newInstance(), config.getBasePath(), config.shouldAssumeDatePartitioning());
+ metaClient.getStorage(), config.getBasePath(), config.shouldAssumeDatePartitioning());
List<String> fsPartitions = fsBackedTableMetadata.getAllPartitionPaths();
List<String> metadataPartitions = tableMetadata.getAllPartitionPaths();
@@ -2801,7 +2801,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// Metadata table has a fixed number of partitions
// Cannot use FSUtils.getAllFoldersWithPartitionMetaFile for this as that function filters all directory
// in the .hoodie folder.
- List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, getMetadataTableBasePath(basePath), false, false);
+ List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, storage, getMetadataTableBasePath(basePath), false, false);
assertEquals(metadataWriter.getEnabledPartitionTypes().size(), metadataTablePartitions.size());
final Map<String, MetadataPartitionType> metadataEnabledPartitionTypes = new HashMap<>();
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
index 0d4b77ec43d..ad92748a15e 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
@@ -1028,7 +1028,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
private Set<String> verifyRecordKeys(List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, List<GenericRecord> records) {
for (WriteStatus status : allStatus) {
StoragePath filePath = new StoragePath(basePath, status.getStat().getPath());
- records.addAll(FileFormatUtils.getInstance(metaClient).readAvroRecords(storageConf, filePath));
+ records.addAll(FileFormatUtils.getInstance(metaClient).readAvroRecords(storage, filePath));
}
Set<String> expectedKeys = recordsToRecordKeySet(expectedRecords);
assertEquals(records.size(), expectedKeys.size());
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
index d14c2a30921..dedf787c127 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
@@ -161,13 +161,13 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storageConf, new StoragePath(filePath.toUri()));
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storage, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
GenericRecord newRecord;
int index = 0;
for (GenericRecord record : fileRecords) {
@@ -202,7 +202,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
// Check whether the record has been updated
Path updatedFilePath = allFiles[0].getPath();
BloomFilter updatedFilter =
- fileUtils.readBloomFilterFromMetadata(storageConf, new StoragePath(updatedFilePath.toUri()));
+ fileUtils.readBloomFilterFromMetadata(storage, new StoragePath(updatedFilePath.toUri()));
for (HoodieRecord record : records) {
// No change to the _row_key
assertTrue(updatedFilter.mightContain(record.getRecordKey()));
@@ -508,13 +508,13 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storageConf, new StoragePath(filePath.toUri()));
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(storage, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
int index = 0;
for (GenericRecord record : fileRecords) {
assertEquals(records.get(index).getRecordKey(), record.get("_row_key").toString());
@@ -533,7 +533,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
filePath = allFiles[0].getPath();
// Read the base file, check the record content
- fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
// Check that the two records are deleted successfully
assertEquals(1, fileRecords.size());
assertEquals(records.get(1).getRecordKey(), fileRecords.get(0).get("_row_key").toString());
@@ -550,7 +550,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
filePath = allFiles[0].getPath();
// Read the base file, check the record content
- fileRecords = fileUtils.readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ fileRecords = fileUtils.readAvroRecords(storage, new StoragePath(filePath.toUri()));
// Check whether all records have been deleted
assertEquals(0, fileRecords.size());
}
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
index 1e43a4d3840..439ab09c897 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
@@ -260,7 +260,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
}
protected HoodieTableMetadata metadata(HoodieWriteConfig clientConfig, HoodieEngineContext engineContext) {
- return HoodieTableMetadata.create(engineContext, clientConfig.getMetadataConfig(), clientConfig.getBasePath());
+ return HoodieTableMetadata.create(engineContext, metaClient.getStorage(), clientConfig.getMetadataConfig(), clientConfig.getBasePath());
}
/**
@@ -405,7 +405,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
// Metadata table has a fixed number of partitions
// Cannot use FSUtils.getAllFoldersWithPartitionMetaFile for this as that function filters all directory
// in the .hoodie folder.
- List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, HoodieTableMetadata.getMetadataTableBasePath(basePath), false, false);
+ List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, storage, HoodieTableMetadata.getMetadataTableBasePath(basePath), false, false);
List<MetadataPartitionType> enabledPartitionTypes = metadataWriter.getEnabledPartitionTypes();
@@ -908,7 +908,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
HashMap<String, String> paths =
getLatestFileIDsToFullPath(basePath, commitTimeline, Arrays.asList(commitInstant));
return paths.values().stream().map(StoragePath::new).flatMap(path ->
- FileFormatUtils.getInstance(path).readAvroRecords(context.getStorageConf(), path).stream())
+ FileFormatUtils.getInstance(path).readAvroRecords(storage, path).stream())
.filter(record -> {
if (filterByCommitTime) {
Object commitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD);
@@ -938,7 +938,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
List<HoodieBaseFile> latestFiles = getLatestBaseFiles(basePath, storage, paths);
return latestFiles.stream().mapToLong(baseFile ->
FileFormatUtils.getInstance(baseFile.getStoragePath())
- .readAvroRecords(context.getStorageConf(), baseFile.getStoragePath()).size())
+ .readAvroRecords(storage, baseFile.getStoragePath()).size())
.sum();
} catch (Exception e) {
throw new HoodieException("Error reading hoodie table as a dataframe", e);
@@ -975,7 +975,8 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
HashMap<String, String> fileIdToFullPath = getLatestFileIDsToFullPath(basePath, commitTimeline, commitsToReturn);
String[] paths = fileIdToFullPath.values().toArray(new String[fileIdToFullPath.size()]);
if (paths[0].endsWith(HoodieFileFormat.PARQUET.getFileExtension())) {
- return Arrays.stream(paths).map(StoragePath::new).flatMap(path -> FileFormatUtils.getInstance(path).readAvroRecords(context.getStorageConf(), path).stream())
+ return Arrays.stream(paths).map(StoragePath::new).flatMap(path -> FileFormatUtils.getInstance(path)
+ .readAvroRecords(storage, path).stream())
.filter(record -> {
if (lastCommitTimeOpt.isPresent()) {
Object commitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
index 3182b2f9a66..976795b7dc6 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
@@ -57,8 +57,10 @@ import org.apache.hudi.io.IOUtils;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.HoodieWriteMetadata;
@@ -304,7 +306,7 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
try {
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(config.getSchema()));
HoodieMergedLogRecordScanner scanner = HoodieMergedLogRecordScanner.newBuilder()
- .withStorage(table.getMetaClient().getStorage())
+ .withStorage(table.getStorage())
.withBasePath(table.getMetaClient().getBasePath())
.withLogFilePaths(clusteringOp.getDeltaFilePaths())
.withReaderSchema(readerSchema)
@@ -380,8 +382,10 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
private HoodieFileReader getBaseOrBootstrapFileReader(StorageConfiguration<?> storageConf, String bootstrapBasePath, Option<String[]> partitionFields, ClusteringOperation clusteringOp)
throws IOException {
- HoodieFileReader baseFileReader = getHoodieSparkIOFactory(storageConf).getReaderFactory(recordType)
- .getFileReader(writeConfig, new StoragePath(clusteringOp.getDataFilePath()));
+ StoragePath dataFilePath = new StoragePath(clusteringOp.getDataFilePath());
+ HoodieStorage storage = new HoodieHadoopStorage(dataFilePath, storageConf);
+ HoodieFileReader baseFileReader = getHoodieSparkIOFactory(storage).getReaderFactory(recordType)
+ .getFileReader(writeConfig, dataFilePath);
// handle bootstrap path
if (StringUtils.nonEmpty(clusteringOp.getBootstrapFilePath()) && StringUtils.nonEmpty(bootstrapBasePath)) {
String bootstrapFilePath = clusteringOp.getBootstrapFilePath();
@@ -392,9 +396,9 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
partitionValues = getPartitionFieldVals(partitionFields, partitionFilePath, bootstrapBasePath, baseFileReader.getSchema(),
storageConf.unwrapAs(Configuration.class));
}
- baseFileReader = getHoodieSparkIOFactory(storageConf).getReaderFactory(recordType).newBootstrapFileReader(
+ baseFileReader = getHoodieSparkIOFactory(storage).getReaderFactory(recordType).newBootstrapFileReader(
baseFileReader,
- getHoodieSparkIOFactory(storageConf).getReaderFactory(recordType).getFileReader(
+ getHoodieSparkIOFactory(storage).getReaderFactory(recordType).getFileReader(
writeConfig, new StoragePath(bootstrapFilePath)), partitionFields,
partitionValues);
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
index a7faca1a418..b0ab5ee7767 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
@@ -147,7 +147,8 @@ public abstract class SingleSparkJobExecutionStrategy<T>
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(getWriteConfig().getSchema()));
Iterable<HoodieRecord<T>> indexedRecords = () -> {
try {
- HoodieFileReader baseFileReader = getHoodieSparkIOFactory(getHoodieTable().getStorageConf()).getReaderFactory(recordType)
+ HoodieFileReader baseFileReader = getHoodieSparkIOFactory(getHoodieTable().getStorage())
+ .getReaderFactory(recordType)
.getFileReader(writeConfig, new StoragePath(clusteringOp.getDataFilePath()));
Option<BaseKeyGenerator> keyGeneratorOp = HoodieSparkKeyGeneratorFactory.createBaseKeyGenerator(writeConfig);
// NOTE: Record have to be cloned here to make sure if it holds low-level engine-specific
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java
index 25fae3cb6f5..95055e582bc 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java
@@ -59,7 +59,7 @@ public abstract class SparkPreCommitValidator<T, I, K, O extends HoodieData<Writ
this.table = table;
this.engineContext = engineContext;
this.writeConfig = writeConfig;
- this.metrics = new HoodieMetrics(writeConfig, engineContext.getStorageConf());
+ this.metrics = new HoodieMetrics(writeConfig, table.getStorage());
}
protected Set<String> getPartitionsModified(HoodieWriteMetadata<O> writeResult) {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
index 59bbbec3dd4..ca3971822ed 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
@@ -28,6 +28,7 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.index.HoodieIndexUtils;
import org.apache.hudi.io.HoodieKeyLookupResult;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.spark.api.java.function.FlatMapFunction;
@@ -127,7 +128,7 @@ public class HoodieFileProbingFunction implements
final HoodieBaseFile dataFile = fileIDBaseFileMap.get(fileId);
List<String> matchingKeys = HoodieIndexUtils.filterKeysFromFile(dataFile.getStoragePath(),
- candidateRecordKeys, storageConf);
+ candidateRecordKeys, HoodieStorageUtils.getStorage(dataFile.getStoragePath(), storageConf));
LOG.debug(
String.format("Bloom filter candidates (%d) / false positives (%d), actual matches (%d)",
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java
index 06b33c8dded..bc3538429e7 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java
@@ -21,7 +21,7 @@ package org.apache.hudi.io.storage;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -31,21 +31,21 @@ import java.io.IOException;
public class HoodieSparkFileReaderFactory extends HoodieFileReaderFactory {
- public HoodieSparkFileReaderFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieSparkFileReaderFactory(HoodieStorage storage) {
+ super(storage);
}
@Override
public HoodieFileReader newParquetFileReader(StoragePath path) {
- storageConf.setIfUnset(SQLConf.PARQUET_BINARY_AS_STRING().key(), SQLConf.PARQUET_BINARY_AS_STRING().defaultValueString());
- storageConf.setIfUnset(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), SQLConf.PARQUET_INT96_AS_TIMESTAMP().defaultValueString());
- storageConf.setIfUnset(SQLConf.CASE_SENSITIVE().key(), SQLConf.CASE_SENSITIVE().defaultValueString());
+ storage.getConf().setIfUnset(SQLConf.PARQUET_BINARY_AS_STRING().key(), SQLConf.PARQUET_BINARY_AS_STRING().defaultValueString());
+ storage.getConf().setIfUnset(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), SQLConf.PARQUET_INT96_AS_TIMESTAMP().defaultValueString());
+ storage.getConf().setIfUnset(SQLConf.CASE_SENSITIVE().key(), SQLConf.CASE_SENSITIVE().defaultValueString());
// Using string value of this conf to preserve compatibility across spark versions.
- storageConf.setIfUnset("spark.sql.legacy.parquet.nanosAsLong", "false");
+ storage.getConf().setIfUnset("spark.sql.legacy.parquet.nanosAsLong", "false");
// This is a required config since Spark 3.4.0: SQLConf.PARQUET_INFER_TIMESTAMP_NTZ_ENABLED
// Using string value of this conf to preserve compatibility across spark versions.
- storageConf.setIfUnset("spark.sql.parquet.inferTimestampNTZ.enabled", "true");
- return new HoodieSparkParquetReader(storageConf, path);
+ storage.getConf().setIfUnset("spark.sql.parquet.inferTimestampNTZ.enabled", "true");
+ return new HoodieSparkParquetReader(storage, path);
}
@Override
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java
index 6a513e2d7d6..eedc560bdaf 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java
@@ -27,6 +27,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.storage.row.HoodieRowParquetConfig;
import org.apache.hudi.io.storage.row.HoodieRowParquetWriteSupport;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
@@ -42,8 +43,8 @@ import java.io.OutputStream;
public class HoodieSparkFileWriterFactory extends HoodieFileWriterFactory {
- public HoodieSparkFileWriterFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieSparkFileWriterFactory(HoodieStorage storage) {
+ super(storage);
}
@Override
@@ -56,14 +57,14 @@ public class HoodieSparkFileWriterFactory extends HoodieFileWriterFactory {
if (compressionCodecName.isEmpty()) {
compressionCodecName = null;
}
- HoodieRowParquetWriteSupport writeSupport = getHoodieRowParquetWriteSupport(storageConf, schema,
+ HoodieRowParquetWriteSupport writeSupport = getHoodieRowParquetWriteSupport(storage.getConf(), schema,
config, enableBloomFilter(populateMetaFields, config));
HoodieRowParquetConfig parquetConfig = new HoodieRowParquetConfig(writeSupport,
CompressionCodecName.fromConf(compressionCodecName),
config.getIntOrDefault(HoodieStorageConfig.PARQUET_BLOCK_SIZE),
config.getIntOrDefault(HoodieStorageConfig.PARQUET_PAGE_SIZE),
config.getLongOrDefault(HoodieStorageConfig.PARQUET_MAX_FILE_SIZE),
- storageConf.unwrapAs(Configuration.class),
+ storage.getConf().unwrapAs(Configuration.class),
config.getDoubleOrDefault(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION),
config.getBooleanOrDefault(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED));
parquetConfig.getHadoopConf().addResource(writeSupport.getHadoopConf());
@@ -74,7 +75,7 @@ public class HoodieSparkFileWriterFactory extends HoodieFileWriterFactory {
protected HoodieFileWriter newParquetFileWriter(OutputStream outputStream, HoodieConfig config,
Schema schema) throws IOException {
boolean enableBloomFilter = false;
- HoodieRowParquetWriteSupport writeSupport = getHoodieRowParquetWriteSupport(storageConf, schema, config, enableBloomFilter);
+ HoodieRowParquetWriteSupport writeSupport = getHoodieRowParquetWriteSupport(storage.getConf(), schema, config, enableBloomFilter);
String compressionCodecName = config.getStringOrDefault(HoodieStorageConfig.PARQUET_COMPRESSION_CODEC_NAME);
// Support PARQUET_COMPRESSION_CODEC_NAME is ""
if (compressionCodecName.isEmpty()) {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java
index 9d673b98908..55a0aad05f2 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java
@@ -20,25 +20,26 @@
package org.apache.hudi.io.storage;
import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.io.hadoop.HoodieHadoopIOFactory;
+import org.apache.hudi.storage.HoodieStorage;
/**
* Creates readers and writers for SPARK and AVRO record payloads
*/
public class HoodieSparkIOFactory extends HoodieHadoopIOFactory {
- public HoodieSparkIOFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieSparkIOFactory(HoodieStorage storage) {
+ super(storage);
}
- public static HoodieSparkIOFactory getHoodieSparkIOFactory(StorageConfiguration<?> storageConf) {
- return new HoodieSparkIOFactory(storageConf);
+ public static HoodieSparkIOFactory getHoodieSparkIOFactory(HoodieStorage storage) {
+ return new HoodieSparkIOFactory(storage);
}
@Override
public HoodieFileReaderFactory getReaderFactory(HoodieRecord.HoodieRecordType recordType) {
if (recordType == HoodieRecord.HoodieRecordType.SPARK) {
- return new HoodieSparkFileReaderFactory(storageConf);
+ return new HoodieSparkFileReaderFactory(storage);
}
return super.getReaderFactory(recordType);
}
@@ -46,7 +47,7 @@ public class HoodieSparkIOFactory extends HoodieHadoopIOFactory {
@Override
public HoodieFileWriterFactory getWriterFactory(HoodieRecord.HoodieRecordType recordType) {
if (recordType == HoodieRecord.HoodieRecordType.SPARK) {
- return new HoodieSparkFileWriterFactory(storageConf);
+ return new HoodieSparkFileWriterFactory(storage);
}
return super.getWriterFactory(recordType);
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java
index 8bbf7840d5b..49b647eec5f 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java
@@ -30,7 +30,7 @@ import org.apache.hudi.common.util.ParquetUtils;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.common.util.collection.CloseableMappingIterator;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -59,31 +59,31 @@ import static org.apache.parquet.avro.AvroSchemaConverter.ADD_LIST_ELEMENT_RECOR
public class HoodieSparkParquetReader implements HoodieSparkFileReader {
private final StoragePath path;
- private final StorageConfiguration<?> conf;
+ private final HoodieStorage storage;
private final FileFormatUtils parquetUtils;
private List<ParquetReaderIterator> readerIterators = new ArrayList<>();
- public HoodieSparkParquetReader(StorageConfiguration<?> conf, StoragePath path) {
+ public HoodieSparkParquetReader(HoodieStorage storage, StoragePath path) {
this.path = path;
- this.conf = conf.newInstance();
+ this.storage = storage.newInstance(path, storage.getConf().newInstance());
// Avoid adding record in list element when convert parquet schema to avro schema
- conf.set(ADD_LIST_ELEMENT_RECORDS, "false");
+ this.storage.getConf().set(ADD_LIST_ELEMENT_RECORDS, "false");
this.parquetUtils = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET);
}
@Override
public String[] readMinMaxRecordKeys() {
- return parquetUtils.readMinMaxRecordKeys(conf, path);
+ return parquetUtils.readMinMaxRecordKeys(storage, path);
}
@Override
public BloomFilter readBloomFilter() {
- return parquetUtils.readBloomFilterFromMetadata(conf, path);
+ return parquetUtils.readBloomFilterFromMetadata(storage, path);
}
@Override
public Set<String> filterRowKeys(Set<String> candidateRowKeys) {
- return parquetUtils.filterRowKeys(conf, path, candidateRowKeys);
+ return parquetUtils.filterRowKeys(storage, path, candidateRowKeys);
}
@Override
@@ -122,12 +122,12 @@ public class HoodieSparkParquetReader implements HoodieSparkFileReader {
}
StructType readerStructType = HoodieInternalRowUtils.getCachedSchema(readerSchema);
StructType requestedStructType = HoodieInternalRowUtils.getCachedSchema(requestedSchema);
- conf.set(ParquetReadSupport.PARQUET_READ_SCHEMA, readerStructType.json());
- conf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA(), requestedStructType.json());
- conf.set(SQLConf.PARQUET_BINARY_AS_STRING().key(), SQLConf.get().getConf(SQLConf.PARQUET_BINARY_AS_STRING()).toString());
- conf.set(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), SQLConf.get().getConf(SQLConf.PARQUET_INT96_AS_TIMESTAMP()).toString());
+ storage.getConf().set(ParquetReadSupport.PARQUET_READ_SCHEMA, readerStructType.json());
+ storage.getConf().set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA(), requestedStructType.json());
+ storage.getConf().set(SQLConf.PARQUET_BINARY_AS_STRING().key(), SQLConf.get().getConf(SQLConf.PARQUET_BINARY_AS_STRING()).toString());
+ storage.getConf().set(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), SQLConf.get().getConf(SQLConf.PARQUET_INT96_AS_TIMESTAMP()).toString());
ParquetReader<InternalRow> reader = ParquetReader.<InternalRow>builder((ReadSupport) new ParquetReadSupport(), new Path(path.toUri()))
- .withConf(conf.unwrapAs(Configuration.class))
+ .withConf(storage.getConf().unwrapAs(Configuration.class))
.build();
ParquetReaderIterator<InternalRow> parquetReaderIterator = new ParquetReaderIterator<>(reader);
readerIterators.add(parquetReaderIterator);
@@ -139,8 +139,8 @@ public class HoodieSparkParquetReader implements HoodieSparkFileReader {
// Some types in avro are not compatible with parquet.
// Avro only supports representing Decimals as fixed byte array
// and therefore if we convert to Avro directly we'll lose logical type-info.
- MessageType messageType = ((ParquetUtils) parquetUtils).readSchema(conf, path);
- StructType structType = new ParquetToSparkSchemaConverter(conf.unwrapAs(Configuration.class)).convert(messageType);
+ MessageType messageType = ((ParquetUtils) parquetUtils).readSchema(storage, path);
+ StructType structType = new ParquetToSparkSchemaConverter(storage.getConf().unwrapAs(Configuration.class)).convert(messageType);
return SparkAdapterSupport$.MODULE$.sparkAdapter()
.getAvroSchemaConverters()
.toAvroType(structType, true, messageType.getName(), StringUtils.EMPTY_STRING);
@@ -153,6 +153,6 @@ public class HoodieSparkParquetReader implements HoodieSparkFileReader {
@Override
public long getTotalRecords() {
- return parquetUtils.getRowCount(conf, path);
+ return parquetUtils.getRowCount(storage, path);
}
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
index 2a8c395d0d5..fb102a36bcb 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
@@ -112,7 +112,7 @@ public class HoodieRowCreateHandle implements Serializable {
this.currTimer = HoodieTimer.start();
- HoodieStorage storage = table.getMetaClient().getStorage();
+ HoodieStorage storage = table.getStorage();
String writeToken = getWriteToken(taskPartitionId, taskId, taskEpochId);
String fileName = FSUtils.makeBaseFileName(instantTime, writeToken, this.fileId,
@@ -238,7 +238,7 @@ public class HoodieRowCreateHandle implements Serializable {
stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT);
stat.setFileId(fileId);
stat.setPath(new StoragePath(writeConfig.getBasePath()), path);
- long fileSizeInBytes = FSUtils.getFileSize(table.getMetaClient().getStorage(), path);
+ long fileSizeInBytes = FSUtils.getFileSize(table.getStorage(), path);
stat.setTotalWriteBytes(fileSizeInBytes);
stat.setFileSizeInBytes(fileSizeInBytes);
stat.setTotalWriteErrors(writeStatus.getTotalErrorRecords());
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
index 34b1c91e07b..39a32296014 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
@@ -109,7 +109,7 @@ public class SparkHoodieBackedTableMetadataWriter extends HoodieBackedTableMetad
} else {
registry = Registry.getRegistry("HoodieMetadata");
}
- this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), storageConf));
+ this.metrics = Option.of(new HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(), dataMetaClient.getStorage()));
} else {
this.metrics = Option.empty();
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java
index 565551505c6..e1d3d803ed0 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java
@@ -82,7 +82,7 @@ class ParquetBootstrapMetadataHandler extends BaseBootstrapMetadataHandler {
Schema schema) throws Exception {
HoodieRecord.HoodieRecordType recordType = table.getConfig().getRecordMerger().getRecordType();
- HoodieFileReader reader = getHoodieSparkIOFactory(table.getStorageConf()).getReaderFactory(recordType)
+ HoodieFileReader reader = getHoodieSparkIOFactory(table.getStorage()).getReaderFactory(recordType)
.getFileReader(table.getConfig(), sourceFilePath);
HoodieExecutor<Void> executor = null;
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
index c51bb5f21c4..843581a36a0 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
@@ -77,7 +77,7 @@ public class SparkDeletePartitionCommitActionExecutor<T>
// created requested
HoodieInstant dropPartitionsInstant =
new HoodieInstant(REQUESTED, REPLACE_COMMIT_ACTION, instantTime);
- if (!table.getMetaClient().getStorage().exists(
+ if (!table.getStorage().exists(
new StoragePath(table.getMetaClient().getMetaPath(),
dropPartitionsInstant.getFileName()))) {
HoodieRequestedReplaceMetadata requestedReplaceMetadata =
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java
index 868c8e2b42a..776ec1dbf1e 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteTableCommitActionExecutor.java
@@ -45,7 +45,8 @@ public class SparkInsertOverwriteTableCommitActionExecutor<T>
@Override
protected Map<String, List<String>> getPartitionToReplacedFileIds(HoodieWriteMetadata<HoodieData<WriteStatus>> writeMetadata) {
- List<String> partitionPaths = FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), table.getMetaClient().getBasePath());
+ List<String> partitionPaths = FSUtils.getAllPartitionPaths(
+ context, table.getStorage(), config.getMetadataConfig(), table.getMetaClient().getBasePath());
if (partitionPaths == null || partitionPaths.isEmpty()) {
return Collections.emptyMap();
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
index 4af761d61d0..dd67cf31e09 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
@@ -136,7 +136,7 @@ public class TestClientRollback extends HoodieClientTestBase {
assertNoWriteErrors(statuses);
HoodieWriteConfig config = getConfig();
List<String> partitionPaths =
- FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), cfg.getBasePath());
+ FSUtils.getAllPartitionPaths(context, storage, config.getMetadataConfig(), cfg.getBasePath());
metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieSparkTable table = HoodieSparkTable.create(getConfig(), context, metaClient);
final BaseFileOnlyView view1 = table.getBaseFileOnlyView();
@@ -317,7 +317,7 @@ public class TestClientRollback extends HoodieClientTestBase {
assertNoWriteErrors(statuses);
HoodieWriteConfig config = getConfig();
List<String> partitionPaths =
- FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), cfg.getBasePath());
+ FSUtils.getAllPartitionPaths(context, storage, config.getMetadataConfig(), cfg.getBasePath());
metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieSparkTable table = HoodieSparkTable.create(getConfig(), context, metaClient);
final BaseFileOnlyView view1 = table.getBaseFileOnlyView();
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java
index 26f3e193469..b96d8723b51 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java
@@ -133,7 +133,7 @@ public class TestUpdateSchemaEvolution extends HoodieSparkClientTestHarness impl
HoodieMergeHandle mergeHandle = new HoodieMergeHandle(updateTable.getConfig(), "101", updateTable,
updateRecords.iterator(), updateRecords.get(0).getPartitionPath(), insertResult.getFileId(), supplier, Option.empty());
List<GenericRecord> oldRecords = FileFormatUtils.getInstance(updateTable.getBaseFileFormat())
- .readAvroRecords(updateTable.getStorageConf(),
+ .readAvroRecords(updateTable.getStorage(),
new StoragePath(updateTable.getConfig().getBasePath() + "/" + insertResult.getStat().getPath()),
mergeHandle.getWriterSchemaWithMetaFields());
for (GenericRecord rec : oldRecords) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
index ae4d8eba5a6..0130fb87956 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
@@ -174,7 +174,8 @@ public class TestExternalPathHandling extends HoodieClientTestBase {
assertFileGroupCorrectness(instantTime3, partitionPath2, filePath3, fileId3, partitionPath2.isEmpty() ? 2 : 1);
// assert that column stats are correct
- HoodieBackedTableMetadata hoodieBackedTableMetadata = new HoodieBackedTableMetadata(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), true);
+ HoodieBackedTableMetadata hoodieBackedTableMetadata = new HoodieBackedTableMetadata(
+ context, metaClient.getStorage(), writeConfig.getMetadataConfig(), writeConfig.getBasePath(), true);
assertEmptyColStats(hoodieBackedTableMetadata, partitionPath1, fileName1);
assertColStats(hoodieBackedTableMetadata, partitionPath1, fileName2);
assertColStats(hoodieBackedTableMetadata, partitionPath2, fileName3);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
index cd568d7fe42..5783eeb60da 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
@@ -170,8 +170,8 @@ import static org.apache.hudi.common.table.timeline.HoodieTimeline.REQUESTED_EXT
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.getNextCommitTime;
import static org.apache.hudi.config.HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS;
-import static org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.METADATA_COMPACTION_TIME_SUFFIX;
import static org.apache.hudi.io.storage.HoodieSparkIOFactory.getHoodieSparkIOFactory;
+import static org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.METADATA_COMPACTION_TIME_SUFFIX;
import static org.apache.hudi.metadata.HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP;
import static org.apache.hudi.metadata.HoodieTableMetadata.getMetadataTableBasePath;
import static org.apache.hudi.metadata.HoodieTableMetadataUtil.deleteMetadataTable;
@@ -762,7 +762,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
metadataWriter.deletePartitions("0000003", Arrays.asList(COLUMN_STATS));
HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
- List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, metadataMetaClient.getBasePath(), false, false);
+ List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, metadataMetaClient.getStorage(), metadataMetaClient.getBasePath(), false, false);
// partition should be physically deleted
assertEquals(metadataWriter.getEnabledPartitionTypes().size(), metadataTablePartitions.size());
assertFalse(metadataTablePartitions.contains(COLUMN_STATS.getPartitionPath()));
@@ -821,7 +821,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
List<FileSlice> fileSlices = table.getSliceView().getLatestFileSlices("files").collect(Collectors.toList());
HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
- getHoodieSparkIOFactory(context.getStorageConf()).getReaderFactory(HoodieRecordType.AVRO).getFileReader(
+ getHoodieSparkIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO).getFileReader(
table.getConfig(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
records.forEach(entry -> {
@@ -1061,7 +1061,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
HoodieTableMetadata metadataReader = HoodieTableMetadata.create(
- context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
Map<String, HoodieRecordGlobalLocation> result = metadataReader
.readRecordIndex(records1.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toList()));
assertEquals(0, result.size(), "RI should not return entries that are rolled back.");
@@ -1354,7 +1354,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
final HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
- HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase) getHoodieSparkIOFactory(context.getStorageConf())
+ HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase) getHoodieSparkIOFactory(storage)
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(table.getConfig(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
@@ -2948,7 +2948,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
doWriteOperation(testTable, HoodieActiveTimeline.createNewInstantTime(1));
doWriteOperation(testTable, HoodieActiveTimeline.createNewInstantTime(1));
// ensure the compaction is triggered and executed
- try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), true)) {
+ try (HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(
+ context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), true)) {
HoodieTableMetaClient metadataMetaClient = metadata.getMetadataMetaClient();
final HoodieActiveTimeline activeTimeline = metadataMetaClient.reloadActiveTimeline();
Option<HoodieInstant> lastCompaction = activeTimeline.filterCompletedInstants()
@@ -3024,7 +3025,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertNoWriteErrors(writeStatuses);
validateMetadata(client);
- Metrics metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storageConf);
+ Metrics metrics = Metrics.getInstance(writeConfig.getMetricsConfig(), storage);
assertTrue(metrics.getRegistry().getGauges().containsKey(HoodieMetadataMetrics.INITIALIZE_STR + ".count"));
assertTrue(metrics.getRegistry().getGauges().containsKey(HoodieMetadataMetrics.INITIALIZE_STR + ".totalDuration"));
assertTrue((Long) metrics.getRegistry().getGauges().get(HoodieMetadataMetrics.INITIALIZE_STR + ".count").getValue() >= 1L);
@@ -3386,7 +3387,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
allRecords.addAll(secondBatchOfrecords);
// RI should have created mappings for all the records inserted above
- HoodieTableMetadata metadataReader = HoodieTableMetadata.create(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ HoodieTableMetadata metadataReader = HoodieTableMetadata.create(context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
Map<String, HoodieRecordGlobalLocation> result = metadataReader
.readRecordIndex(allRecords.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toList()));
assertEquals(allRecords.size(), result.size(), "RI should have mapping for all the records in firstCommit");
@@ -3401,7 +3402,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
client.delete(jsc.parallelize(recordsToDelete, 1).map(HoodieRecord::getKey), deleteTime);
// RI should not return mappings for deleted records
- metadataReader = HoodieTableMetadata.create(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ metadataReader = HoodieTableMetadata.create(
+ context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
result = metadataReader.readRecordIndex(allRecords.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toList()));
assertEquals(allRecords.size() - recordsToDelete.size(), result.size(), "RI should not have mapping for deleted records");
result.keySet().forEach(mappingKey -> assertFalse(keysToDelete.contains(mappingKey), "RI should not have mapping for deleted records"));
@@ -3414,11 +3416,12 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
String deleteTime = client.startCommit();
client.delete(jsc.emptyRDD(), deleteTime);
- HoodieTableMetadata metadataReader = HoodieTableMetadata.create(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ HoodieTableMetadata metadataReader = HoodieTableMetadata.create(
+ context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
assertTrue(metadataReader.getLatestCompactionTime().isPresent(), "Compaction should have taken place on MDT");
// RI should not return mappings for deleted records
- metadataReader = HoodieTableMetadata.create(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ metadataReader = HoodieTableMetadata.create(context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
Map<String, HoodieRecordGlobalLocation> result = metadataReader.readRecordIndex(allRecords.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toList()));
assertEquals(allRecords.size() - keysToDelete.size(), result.size(), "RI should not have mapping for deleted records");
result.keySet().forEach(mappingKey -> assertFalse(keysToDelete.contains(mappingKey), "RI should not have mapping for deleted records"));
@@ -3428,7 +3431,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
client.upsert(jsc.parallelize(recordsToDelete, 1), reinsertTime).collect();
// New mappings should have been created for re-inserted records and should map to the new commit time
- metadataReader = HoodieTableMetadata.create(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
+ metadataReader = HoodieTableMetadata.create(context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath());
result = metadataReader.readRecordIndex(allRecords.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toList()));
assertEquals(allRecords.size(), result.size(), "RI should have mappings for re-inserted records");
for (String reInsertedKey : keysToDelete) {
@@ -3484,7 +3487,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// Metadata table has a fixed number of partitions
// Cannot use FSUtils.getAllFoldersWithPartitionMetaFile for this as that function filters all directory
// in the .hoodie folder.
- List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, getMetadataTableBasePath(basePath),
+ List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, storage, getMetadataTableBasePath(basePath),
false, false);
assertEquals(metadataWriter.getEnabledPartitionTypes().size(), metadataTablePartitions.size());
@@ -3523,7 +3526,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// Partitions should match
FileSystemBackedTableMetadata fsBackedTableMetadata = new FileSystemBackedTableMetadata(engineContext, metaClient.getTableConfig(),
- metaClient.getStorageConf(), config.getBasePath(), config.shouldAssumeDatePartitioning());
+ metaClient.getStorage(), config.getBasePath(), config.shouldAssumeDatePartitioning());
List<String> fsPartitions = fsBackedTableMetadata.getAllPartitionPaths();
List<String> metadataPartitions = tableMetadata.getAllPartitionPaths();
@@ -3691,9 +3694,14 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
.create(storageConf, client.getConfig(), new HoodieSparkEngineContext(jsc));
}
- public static HoodieTableMetadata metadata(SparkRDDWriteClient client) {
+ public static HoodieTableMetadata metadata(SparkRDDWriteClient client, HoodieStorage storage) {
+ HoodieWriteConfig clientConfig = client.getConfig();
+ return HoodieTableMetadata.create(client.getEngineContext(), storage, clientConfig.getMetadataConfig(), clientConfig.getBasePath());
+ }
+
+ public HoodieTableMetadata metadata(SparkRDDWriteClient client) {
HoodieWriteConfig clientConfig = client.getConfig();
- return HoodieTableMetadata.create(client.getEngineContext(), clientConfig.getMetadataConfig(), clientConfig.getBasePath());
+ return HoodieTableMetadata.create(client.getEngineContext(), storage, clientConfig.getMetadataConfig(), clientConfig.getBasePath());
}
private void changeTableVersion(HoodieTableVersion version) throws IOException {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
index e5824b02b03..8222bf2075a 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
@@ -124,7 +124,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
HoodieTableType tableType = HoodieTableType.COPY_ON_WRITE;
init(tableType);
testTable.doWriteOperation("000001", INSERT, emptyList(), asList("p1"), 1);
- HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), reuse);
+ HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), reuse);
assertTrue(tableMetadata.enabled());
List<String> metadataPartitions = tableMetadata.getAllPartitionPaths();
String partition = metadataPartitions.get(0);
@@ -168,7 +168,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
private void verifyBaseMetadataTable(boolean reuseMetadataReaders) throws IOException {
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(
- context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), reuseMetadataReaders);
+ context, storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), reuseMetadataReaders);
assertTrue(tableMetadata.enabled());
List<java.nio.file.Path> fsPartitionPaths = testTable.getAllPartitionPaths();
List<String> fsPartitions = new ArrayList<>();
@@ -211,7 +211,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
init(tableType);
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(context,
- writeConfig.getMetadataConfig(), writeConfig.getBasePath(), false);
+ storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), false);
assertEquals(HoodieTableMetadataKeyGenerator.class.getCanonicalName(),
tableMetadata.getMetadataMetaClient().getTableConfig().getKeyGeneratorClassName());
@@ -225,7 +225,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
public void testNotExistPartition(final HoodieTableType tableType) throws Exception {
init(tableType);
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(context,
- writeConfig.getMetadataConfig(), writeConfig.getBasePath(), false);
+ storage, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), false);
List<StoragePathInfo> allFilesInPartition = tableMetadata.getAllFilesInPartition(
new StoragePath(writeConfig.getBasePath() + "dummy"));
assertEquals(allFilesInPartition.size(), 0);
@@ -382,6 +382,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
private Set<String> getFilePathsInPartition(String partition) throws IOException {
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(
new HoodieLocalEngineContext(storageConf),
+ storage,
HoodieMetadataConfig.newBuilder().enable(true).build(),
basePath);
return tableMetadata.getAllFilesInPartition(new StoragePath(basePath, partition))
@@ -527,7 +528,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
final HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
- getHoodieSparkIOFactory(context.getStorageConf())
+ getHoodieSparkIOFactory(storage)
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(table.getConfig(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index 74e998349ea..1738414f099 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -1213,7 +1213,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
String file1 = statuses.get(0).getFileId();
assertEquals(100,
- fileUtils.readRowKeys(storageConf, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
+ fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
.size(), "file should contain 100 records");
// Update + Inserts such that they just expand file1
@@ -1233,10 +1233,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(file1, statuses.get(0).getFileId(), "Existing file should be expanded");
assertEquals(commitTime1, statuses.get(0).getStat().getPrevCommit(), "Existing file should be expanded");
StoragePath newFile = new StoragePath(basePath, statuses.get(0).getStat().getPath());
- assertEquals(140, fileUtils.readRowKeys(storageConf, newFile).size(),
+ assertEquals(140, fileUtils.readRowKeys(storage, newFile).size(),
"file should contain 140 records");
- List<GenericRecord> records = fileUtils.readAvroRecords(storageConf, newFile);
+ List<GenericRecord> records = fileUtils.readAvroRecords(storage, newFile);
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
assertEquals(commitTime2, record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString(), "only expect commit2");
@@ -1267,7 +1267,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
for (HoodieBaseFile file : files) {
if (file.getFileName().contains(file1)) {
assertEquals(commitTime3, file.getCommitTime(), "Existing file should be expanded");
- records = fileUtils.readAvroRecords(storageConf, new StoragePath(file.getPath()));
+ records = fileUtils.readAvroRecords(storage, new StoragePath(file.getPath()));
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
String recordCommitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
@@ -1283,7 +1283,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(0, keys2.size(), "All keys added in commit 2 must be updated in commit3 correctly");
} else {
assertEquals(commitTime3, file.getCommitTime(), "New file must be written for commit 3");
- records = fileUtils.readAvroRecords(storageConf, new StoragePath(file.getPath()));
+ records = fileUtils.readAvroRecords(storage, new StoragePath(file.getPath()));
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
assertEquals(commitTime3, record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString(),
@@ -1324,7 +1324,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
String file1 = statuses.get(0).getFileId();
assertEquals(100,
- fileUtils.readRowKeys(storageConf, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
+ fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
.size(), "file should contain 100 records");
// Second, set of Inserts should just expand file1
@@ -1340,9 +1340,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(commitTime1, statuses.get(0).getStat().getPrevCommit(), "Existing file should be expanded");
StoragePath newFile = new StoragePath(basePath, statuses.get(0).getStat().getPath());
- assertEquals(140, fileUtils.readRowKeys(storageConf, newFile).size(),
+ assertEquals(140, fileUtils.readRowKeys(storage, newFile).size(),
"file should contain 140 records");
- List<GenericRecord> records = fileUtils.readAvroRecords(storageConf, newFile);
+ List<GenericRecord> records = fileUtils.readAvroRecords(storage, newFile);
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
String recCommitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
@@ -1361,8 +1361,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertNoWriteErrors(statuses);
assertEquals(2, statuses.size(), "2 files needs to be committed.");
assertEquals(340,
- fileUtils.readRowKeys(storageConf, new StoragePath(basePath, statuses.get(0).getStat().getPath())).size()
- + fileUtils.readRowKeys(storageConf, new StoragePath(basePath, statuses.get(1).getStat().getPath())).size(),
+ fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(0).getStat().getPath())).size()
+ + fileUtils.readRowKeys(storage, new StoragePath(basePath, statuses.get(1).getStat().getPath())).size(),
"file should contain 340 records");
HoodieTableMetaClient metaClient = createMetaClient(basePath);
@@ -1374,7 +1374,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
int totalInserts = 0;
for (HoodieBaseFile file : files) {
assertEquals(commitTime3, file.getCommitTime(), "All files must be at commit 3");
- totalInserts += fileUtils.readAvroRecords(storageConf, new StoragePath(file.getPath())).size();
+ totalInserts += fileUtils.readAvroRecords(storage, new StoragePath(file.getPath())).size();
}
assertEquals(totalInserts, inserts1.size() + inserts2.size() + inserts3.size(), "Total number of records must add up");
}
@@ -1408,7 +1408,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
String file1 = statuses.get(0).getFileId();
assertEquals(100,
- FileFormatUtils.getInstance(metaClient).readRowKeys(storageConf, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
+ FileFormatUtils.getInstance(metaClient).readRowKeys(storage, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
.size(), "file should contain 100 records");
// Delete 20 among 100 inserted
@@ -2090,7 +2090,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
private Set<String> verifyRecordKeys(List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, List<GenericRecord> records) {
for (WriteStatus status : allStatus) {
StoragePath filePath = new StoragePath(basePath, status.getStat().getPath());
- records.addAll(FileFormatUtils.getInstance(metaClient).readAvroRecords(storageConf, filePath));
+ records.addAll(FileFormatUtils.getInstance(metaClient).readAvroRecords(storage, filePath));
}
Set<String> expectedKeys = recordsToRecordKeySet(expectedRecords);
assertEquals(records.size(), expectedKeys.size());
@@ -2179,10 +2179,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
StoragePath newFile = new StoragePath(basePath, statuses.get(0).getStat().getPath());
assertEquals(expectedRecords,
- FileFormatUtils.getInstance(metaClient).readRowKeys(storageConf, newFile).size(),
+ FileFormatUtils.getInstance(metaClient).readRowKeys(storage, newFile).size(),
"file should contain 110 records");
- List<GenericRecord> records = FileFormatUtils.getInstance(metaClient).readAvroRecords(storageConf, newFile);
+ List<GenericRecord> records = FileFormatUtils.getInstance(metaClient).readAvroRecords(storage, newFile);
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
assertTrue(keys.contains(recordKey), "key expected to be part of " + instantTime);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
index 1710263bc44..e08d703e386 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
@@ -120,7 +120,7 @@ public class TestRemoteFileSystemViewWithMetadataTable extends HoodieSparkClient
context, config.getViewStorageConfig(),
config.getCommonConfig(),
metaClient -> new HoodieBackedTestDelayedTableMetadata(
- context, config.getMetadataConfig(), metaClient.getBasePathV2().toString(), true)));
+ context, metaClient.getStorage(), config.getMetadataConfig(), metaClient.getBasePathV2().toString(), true)));
timelineService.startService();
timelineServicePort = timelineService.getServerPort();
LOG.info("Started timeline server on port: " + timelineServicePort);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java
index cbbdf5fbea1..de8c3d7d49d 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java
@@ -337,7 +337,7 @@ public class TestHoodieBloomIndex extends TestHoodieMetadataBase {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).build();
HoodieSparkTable table = HoodieSparkTable.create(config, context, metaClient);
List<String> results = HoodieIndexUtils.filterKeysFromFile(
- new StoragePath(Paths.get(basePath, partition, filename).toString()), uuids, storageConf);
+ new StoragePath(Paths.get(basePath, partition, filename).toString()), uuids, storage);
assertEquals(results.size(), 2);
assertTrue(results.get(0).equals("1eb5b87a-1feh-4edd-87b4-6ec96dc405a0")
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java
index 74826c6f39b..f46fe9f6bcb 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java
@@ -55,7 +55,7 @@ public class TestHoodieAvroFileWriterFactory extends HoodieClientTestBase {
HoodieTable table = HoodieSparkTable.create(cfg, context, metaClient);
SparkTaskContextSupplier supplier = new SparkTaskContextSupplier();
HoodieFileWriter parquetWriter = HoodieFileWriterFactory.getFileWriter(instantTime,
- parquetPath, table.getStorageConf(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
+ parquetPath, table.getStorage(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
assertTrue(parquetWriter instanceof HoodieAvroParquetWriter);
parquetWriter.close();
@@ -63,7 +63,7 @@ public class TestHoodieAvroFileWriterFactory extends HoodieClientTestBase {
final StoragePath hfilePath = new StoragePath(
basePath + "/partition/path/f1_1-0-1_000.hfile");
HoodieFileWriter hfileWriter = HoodieFileWriterFactory.getFileWriter(instantTime,
- hfilePath, table.getStorageConf(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
+ hfilePath, table.getStorage(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
assertTrue(hfileWriter instanceof HoodieAvroHFileWriter);
hfileWriter.close();
@@ -71,7 +71,7 @@ public class TestHoodieAvroFileWriterFactory extends HoodieClientTestBase {
final StoragePath orcPath = new StoragePath(
basePath + "/partition/path/f1_1-0-1_000.orc");
HoodieFileWriter orcFileWriter = HoodieFileWriterFactory.getFileWriter(instantTime,
- orcPath, table.getStorageConf(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
+ orcPath, table.getStorage(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
assertTrue(orcFileWriter instanceof HoodieAvroOrcWriter);
orcFileWriter.close();
@@ -80,7 +80,7 @@ public class TestHoodieAvroFileWriterFactory extends HoodieClientTestBase {
basePath + "/partition/path/f.b51192a8-574b-4a85-b246-bcfec03ac8bf_100.log.2_1-0-1");
final Throwable thrown = assertThrows(UnsupportedOperationException.class, () -> {
HoodieFileWriterFactory.getFileWriter(instantTime, logPath,
- table.getStorageConf(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
+ table.getStorage(), cfg.getStorageConfig(), HoodieTestDataGenerator.AVRO_SCHEMA, supplier, HoodieRecordType.AVRO);
}, "should fail since log storage writer is not supported yet.");
assertTrue(thrown.getMessage().contains("format not supported yet."));
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
index c71a0ca85fb..285383db036 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
@@ -206,14 +206,14 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestBase implemen
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
BloomFilter filter = FileFormatUtils.getInstance(table.getBaseFileFormat())
- .readBloomFilterFromMetadata(storageConf, new StoragePath(filePath.toUri()));
+ .readBloomFilterFromMetadata(storage, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
// Read the base file, check the record content
List<GenericRecord> fileRecords = FileFormatUtils.getInstance(table.getBaseFileFormat())
- .readAvroRecords(storageConf, new StoragePath(filePath.toUri()));
+ .readAvroRecords(storage, new StoragePath(filePath.toUri()));
GenericRecord newRecord;
int index = 0;
for (GenericRecord record : fileRecords) {
@@ -248,7 +248,7 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestBase implemen
// Check whether the record has been updated
Path updatedFilePath = allFiles[0].getPath();
BloomFilter updatedFilter =
- FileFormatUtils.getInstance(metaClient).readBloomFilterFromMetadata(storageConf, new StoragePath(updatedFilePath.toUri()));
+ FileFormatUtils.getInstance(metaClient).readBloomFilterFromMetadata(storage, new StoragePath(updatedFilePath.toUri()));
for (HoodieRecord record : records) {
// No change to the _row_key
assertTrue(updatedFilter.mightContain(record.getRecordKey()));
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkRollback.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkRollback.java
index bd13d959732..0a7e5d69291 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkRollback.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkRollback.java
@@ -132,7 +132,7 @@ public class TestHoodieSparkRollback extends SparkClientFunctionalTestHarness {
//validate that metadata table file listing matches reality
metaClient = HoodieTableMetaClient.reload(metaClient);
TestHoodieBackedMetadata.validateMetadata(getConfigToTestMDTRollbacks(true), Option.empty(), fs(), basePath, metaClient,
- storageConf().unwrap(), new HoodieSparkEngineContext(jsc()), TestHoodieBackedMetadata.metadata(client));
+ storageConf().unwrap(), new HoodieSparkEngineContext(jsc()), TestHoodieBackedMetadata.metadata(client, hoodieStorage()));
}
/**
@@ -194,7 +194,8 @@ public class TestHoodieSparkRollback extends SparkClientFunctionalTestHarness {
updateRecords(client, dataGen, "004", records);
//validate that metadata table file listing matches reality
metaClient = HoodieTableMetaClient.reload(metaClient);
- TestHoodieBackedMetadata.validateMetadata(cfg, Option.empty(), fs(), basePath, metaClient, storageConf().unwrap(), new HoodieSparkEngineContext(jsc()), TestHoodieBackedMetadata.metadata(client));
+ TestHoodieBackedMetadata.validateMetadata(cfg, Option.empty(), fs(), basePath, metaClient, storageConf().unwrap(), new HoodieSparkEngineContext(jsc()),
+ TestHoodieBackedMetadata.metadata(client, hoodieStorage()));
}
private void copyOut(HoodieTableType tableType, String commitTime) throws IOException {
@@ -261,7 +262,7 @@ public class TestHoodieSparkRollback extends SparkClientFunctionalTestHarness {
//validate that metadata table file listing matches reality
metaClient = HoodieTableMetaClient.reload(metaClient);
TestHoodieBackedMetadata.validateMetadata(cfg, Option.empty(), fs(), basePath, metaClient,
- storageConf().unwrap(), new HoodieSparkEngineContext(jsc()), TestHoodieBackedMetadata.metadata(client));
+ storageConf().unwrap(), new HoodieSparkEngineContext(jsc()), TestHoodieBackedMetadata.metadata(client, hoodieStorage()));
}
/**
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkers.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkers.java
index ec503a8774c..6c5a3d234c9 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkers.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkers.java
@@ -21,7 +21,6 @@ package org.apache.hudi.table.marker;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.util.CollectionUtils;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.testutils.HoodieClientTestUtils;
@@ -47,7 +46,7 @@ public class TestDirectWriteMarkers extends TestWriteMarkersBase {
this.jsc = new JavaSparkContext(
HoodieClientTestUtils.getSparkConfForTest(TestDirectWriteMarkers.class.getName()));
this.context = new HoodieSparkEngineContext(jsc);
- this.storage = HoodieStorageUtils.getStorage(metaClient.getBasePathV2(), metaClient.getStorageConf());
+ this.storage = metaClient.getStorage();
this.markerFolderPath = new StoragePath(Paths.get(metaClient.getMarkerFolderPath("000")).toUri());
this.writeMarkers = new DirectWriteMarkers(
storage, metaClient.getBasePathV2().toString(), markerFolderPath.toString(), "000");
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java
index 9d6cf92b99d..b01cc9af9ff 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java
@@ -28,7 +28,6 @@ import org.apache.hudi.common.table.view.FileSystemViewStorageType;
import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.MarkerUtils;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.testutils.HoodieClientTestUtils;
import org.apache.hudi.timeline.service.TimelineService;
@@ -59,7 +58,7 @@ public class TestTimelineServerBasedWriteMarkers extends TestWriteMarkersBase {
this.jsc = new JavaSparkContext(
HoodieClientTestUtils.getSparkConfForTest(TestTimelineServerBasedWriteMarkers.class.getName()));
this.context = new HoodieSparkEngineContext(jsc);
- this.storage = HoodieStorageUtils.getStorage(metaClient.getBasePathV2(), metaClient.getStorageConf());
+ this.storage = metaClient.getStorage();
this.markerFolderPath = new StoragePath(metaClient.getMarkerFolderPath("000"));
FileSystemViewStorageConfig storageConf =
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
index e25db7d5924..24ec69d941c 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
@@ -207,7 +207,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
List<String> markerPaths = new ArrayList<>(writeMarkers.allMarkerFilePaths());
if (deletePartialMarkerFiles) {
String toDeleteMarkerFile = markerPaths.get(0);
- table.getMetaClient().getStorage().deleteDirectory(new StoragePath(
+ table.getStorage().deleteDirectory(new StoragePath(
table.getMetaClient().getTempFolderPath() + "/" + commitInstant.getTimestamp()
+ "/" + toDeleteMarkerFile));
markerPaths.remove(toDeleteMarkerFile);
@@ -621,7 +621,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
List<String> markerPaths = new ArrayList<>(writeMarkers.allMarkerFilePaths());
if (deletePartialMarkerFiles) {
String toDeleteMarkerFile = markerPaths.get(0);
- table.getMetaClient().getStorage().deleteDirectory(new StoragePath(
+ table.getStorage().deleteDirectory(new StoragePath(
table.getMetaClient().getTempFolderPath() + "/" + commitInstant.getTimestamp()
+ "/" + toDeleteMarkerFile));
markerPaths.remove(toDeleteMarkerFile);
@@ -660,7 +660,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
assertTrue(writeMarkers.doesMarkerDirExist());
assertEquals(0, getTimelineServerBasedMarkerFileCount(
table.getMetaClient().getMarkerFolderPath(commitInstant.getTimestamp()),
- (FileSystem) table.getMetaClient().getStorage().getFileSystem()));
+ (FileSystem) table.getStorage().getFileSystem()));
} else {
assertFalse(writeMarkers.doesMarkerDirExist());
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java
index 488b7e170d5..c95f71dc127 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java
@@ -29,7 +29,6 @@ import org.apache.hudi.common.testutils.minicluster.HdfsTestService;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -178,7 +177,7 @@ public class FunctionalTestHarness implements SparkProvider, DFSProvider, Hoodie
@AfterAll
public static synchronized void cleanUpAfterAll() throws IOException {
StoragePath workDir = new StoragePath("/tmp");
- HoodieStorage storage = HoodieStorageUtils.getStorage(
+ HoodieStorage storage = new HoodieHadoopStorage(
workDir, HadoopFSUtils.getStorageConf(hdfsTestService.getHadoopConf()));
List<StoragePathInfo> pathInfoList = storage.listDirectEntries(workDir);
for (StoragePathInfo f : pathInfoList) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
index ce089b713dc..5d5194ee449 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
@@ -554,7 +554,7 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
public HoodieTableMetadata metadata(HoodieWriteConfig clientConfig,
HoodieEngineContext hoodieEngineContext) {
return HoodieTableMetadata.create(
- hoodieEngineContext, clientConfig.getMetadataConfig(), clientConfig.getBasePath());
+ hoodieEngineContext, storage, clientConfig.getMetadataConfig(), clientConfig.getBasePath());
}
protected void validateFilesPerPartition(HoodieTestTable testTable,
@@ -634,7 +634,7 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
// Metadata table has a fixed number of partitions
// Cannot use FSUtils.getAllFoldersWithPartitionMetaFile for this as that function filters all directory
// in the .hoodie folder.
- List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, HoodieTableMetadata.getMetadataTableBasePath(basePath),
+ List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, storage, HoodieTableMetadata.getMetadataTableBasePath(basePath),
false, false);
List<MetadataPartitionType> enabledPartitionTypes = metadataWriter.getEnabledPartitionTypes();
diff --git a/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java b/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java
index 9a0eb0ec578..df431f438ef 100644
--- a/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java
+++ b/hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java
@@ -42,6 +42,7 @@ import org.apache.hudi.expression.Expression;
import org.apache.hudi.internal.schema.Types;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -424,7 +425,7 @@ public abstract class BaseHoodieTableFileIndex implements AutoCloseable {
private void doRefresh() {
HoodieTimer timer = HoodieTimer.start();
- resetTableMetadata(createMetadataTable(engineContext, metadataConfig, basePath));
+ resetTableMetadata(createMetadataTable(engineContext, metaClient.getStorage(), metadataConfig, basePath));
// Make sure we reload active timeline
metaClient.reloadActiveTimeline();
@@ -503,10 +504,12 @@ public abstract class BaseHoodieTableFileIndex implements AutoCloseable {
private static HoodieTableMetadata createMetadataTable(
HoodieEngineContext engineContext,
+ HoodieStorage storage,
HoodieMetadataConfig metadataConfig,
StoragePath basePath
) {
- HoodieTableMetadata newTableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, basePath.toString(), true);
+ HoodieTableMetadata newTableMetadata = HoodieTableMetadata.create(
+ engineContext, storage, metadataConfig, basePath.toString(), true);
return newTableMetadata;
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
index 0309aee00a9..a595dcc17de 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
@@ -235,16 +235,22 @@ public class HoodieStorageConfig extends HoodieConfig {
+ "and it is loaded at runtime. This is only required when trying to "
+ "override the existing write context when `hoodie.datasource.write.row.writer.enable=true`.");
+ public static final ConfigProperty<String> HOODIE_STORAGE_CLASS = ConfigProperty
+ .key("hoodie.storage.class")
+ .defaultValue("org.apache.hudi.storage.hadoop.HoodieHadoopStorage")
+ .markAdvanced()
+ .sinceVersion("0.15.0")
+ .withDocumentation("The fully-qualified class name of the `HoodieStorage` implementation class to instantiate. "
+ + "The provided class should implement `org.apache.hudi.storage.HoodieStorage`");
+
public static final ConfigProperty<String> HOODIE_IO_FACTORY_CLASS = ConfigProperty
.key("hoodie.io.factory.class")
- .defaultValue("org.apache.hudi.io.storage.HoodieHadoopIOFactory")
+ .defaultValue("org.apache.hudi.io.hadoop.HoodieHadoopIOFactory")
.markAdvanced()
.sinceVersion("0.15.0")
.withDocumentation("The fully-qualified class name of the factory class to return readers and writers of files used "
+ "by Hudi. The provided class should implement `org.apache.hudi.io.storage.HoodieIOFactory`.");
-
-
/**
* @deprecated Use {@link #PARQUET_MAX_FILE_SIZE} and its methods instead
*/
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
index 30c968d080d..f06db824ab3 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
@@ -219,14 +219,16 @@ public class FSUtils {
}
}
- public static List<String> getAllPartitionPaths(HoodieEngineContext engineContext, String basePathStr,
+ public static List<String> getAllPartitionPaths(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ String basePathStr,
boolean useFileListingFromMetadata,
boolean assumeDatePartitioning) {
HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder()
.enable(useFileListingFromMetadata)
.withAssumeDatePartitioning(assumeDatePartitioning)
.build();
- try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, basePathStr)) {
+ try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, storage, metadataConfig, basePathStr)) {
return tableMetadata.getAllPartitionPaths();
} catch (Exception e) {
throw new HoodieException("Error fetching partition paths from metadata table", e);
@@ -234,9 +236,10 @@ public class FSUtils {
}
public static List<String> getAllPartitionPaths(HoodieEngineContext engineContext,
+ HoodieStorage storage,
HoodieMetadataConfig metadataConfig,
String basePathStr) {
- try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig,
+ try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, storage, metadataConfig,
basePathStr)) {
return tableMetadata.getAllPartitionPaths();
} catch (Exception e) {
@@ -245,10 +248,11 @@ public class FSUtils {
}
public static Map<String, List<StoragePathInfo>> getFilesInPartitions(HoodieEngineContext engineContext,
+ HoodieStorage storage,
HoodieMetadataConfig metadataConfig,
String basePathStr,
String[] partitionPaths) {
- try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig,
+ try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, storage, metadataConfig,
basePathStr)) {
return tableMetadata.getAllFilesInPartitions(Arrays.asList(partitionPaths));
} catch (Exception ex) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java
index 52c6168f0db..1eecf9d3d58 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java
@@ -23,8 +23,7 @@ import org.apache.hudi.common.util.JsonUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -172,11 +171,11 @@ public class HoodieCommitMetadata implements Serializable {
* been touched multiple times in the given commits, the return value will keep the one
* from the latest commit.
*
- * @param storageConf storage configuration.
+ * @param storage {@link HoodieStorage} instance.
* @param basePath The base path
* @return the file full path to file status mapping
*/
- public Map<String, StoragePathInfo> getFullPathToInfo(StorageConfiguration<?> storageConf,
+ public Map<String, StoragePathInfo> getFullPathToInfo(HoodieStorage storage,
String basePath) {
Map<String, StoragePathInfo> fullPathToInfoMap = new HashMap<>();
for (List<HoodieWriteStat> stats : getPartitionToWriteStats().values()) {
@@ -186,8 +185,7 @@ public class HoodieCommitMetadata implements Serializable {
StoragePath fullPath = relativeFilePath != null
? FSUtils.constructAbsolutePath(basePath, relativeFilePath) : null;
if (fullPath != null) {
- long blockSize =
- HoodieStorageUtils.getStorage(fullPath.toString(), storageConf).getDefaultBlockSize(fullPath);
+ long blockSize = storage.getDefaultBlockSize(fullPath);
StoragePathInfo pathInfo = new StoragePathInfo(
fullPath, stat.getFileSizeInBytes(), false, (short) 0, blockSize, 0);
fullPathToInfoMap.put(fullPath.getName(), pathInfo);
@@ -202,16 +200,14 @@ public class HoodieCommitMetadata implements Serializable {
* been touched multiple times in the given commits, the return value will keep the one
* from the latest commit by file group ID.
*
- * <p>Note: different with {@link #getFullPathToInfo(StorageConfiguration, String)},
+ * <p>Note: different with {@link #getFullPathToInfo(HoodieStorage, String)},
* only the latest commit file for a file group is returned,
* this is an optimization for COPY_ON_WRITE table to eliminate legacy files for filesystem view.
*
- * @param storageConf storage configuration.
* @param basePath The base path
* @return the file ID to file status mapping
*/
- public Map<String, StoragePathInfo> getFileIdToInfo(StorageConfiguration<?> storageConf,
- String basePath) {
+ public Map<String, StoragePathInfo> getFileIdToInfo(String basePath) {
Map<String, StoragePathInfo> fileIdToInfoMap = new HashMap<>();
for (List<HoodieWriteStat> stats : getPartitionToWriteStats().values()) {
// Iterate through all the written files.
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
index 5d75414c6ff..9256e6f4440 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
@@ -188,7 +188,7 @@ public class HoodiePartitionMetadata {
FileFormatUtils reader = FileFormatUtils.getInstance(metafilePath);
// Data file format
Map<String, String> metadata = reader.readFooter(
- storage.getConf(), true, metafilePath, PARTITION_DEPTH_KEY, COMMIT_TIME_KEY);
+ storage, true, metafilePath, PARTITION_DEPTH_KEY, COMMIT_TIME_KEY);
props.clear();
props.putAll(metadata);
format = Option.of(reader.getFormat());
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
index 436a8c221fe..42d8cecffc3 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -297,13 +297,12 @@ public class HoodieTableMetaClient implements Serializable {
public HoodieStorage getStorage() {
if (storage == null) {
+ HoodieStorage newStorage = HoodieStorageUtils.getStorage(metaPath, getStorageConf());
ConsistencyGuard consistencyGuard = consistencyGuardConfig.isConsistencyCheckEnabled()
- ? new FailSafeConsistencyGuard(
- HoodieStorageUtils.getStorage(metaPath, getStorageConf()),
- consistencyGuardConfig)
+ ? new FailSafeConsistencyGuard(newStorage, consistencyGuardConfig)
: new NoOpConsistencyGuard();
- storage = getIOFactory(getStorageConf()).getStorage(metaPath,
+ storage = getIOFactory(newStorage).getStorage(metaPath,
fileSystemRetryConfig.isFileSystemActionRetryEnable(),
fileSystemRetryConfig.getMaxRetryIntervalMs(),
fileSystemRetryConfig.getMaxRetryNumbers(),
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
index d0a395c83a0..bf77a712c58 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
@@ -302,7 +302,7 @@ public class TableSchemaResolver {
.orElseThrow(() -> new IllegalArgumentException("Could not find any data file written for compaction "
+ lastCompactionCommit + ", could not get schema for table " + metaClient.getBasePath()));
StoragePath path = new StoragePath(filePath);
- return FileFormatUtils.getInstance(path).readAvroSchema(metaClient.getStorageConf(), path);
+ return FileFormatUtils.getInstance(path).readAvroSchema(metaClient.getStorage(), path);
}
private Schema readSchemaFromLogFile(StoragePath path) throws IOException {
@@ -469,7 +469,7 @@ public class TableSchemaResolver {
// this is a log file
schema = readSchemaFromLogFile(filePath);
} else {
- schema = FileFormatUtils.getInstance(filePath).readAvroSchema(metaClient.getStorageConf(), filePath);
+ schema = FileFormatUtils.getInstance(filePath).readAvroSchema(metaClient.getStorage(), filePath);
}
}
return schema;
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java
index 98227ef1074..108a5b47aee 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java
@@ -40,7 +40,6 @@ import org.apache.hudi.internal.schema.InternalSchema;
import org.apache.hudi.io.SeekableDataInputStream;
import org.apache.hudi.io.util.IOUtils;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StorageSchemes;
@@ -72,7 +71,6 @@ public class HoodieLogFileReader implements HoodieLogFormat.Reader {
private static final String REVERSE_LOG_READER_HAS_NOT_BEEN_ENABLED = "Reverse log reader has not been enabled";
private final HoodieStorage storage;
- private final StorageConfiguration<?> storageConf;
private final HoodieLogFile logFile;
private int bufferSize;
private final byte[] magicBuffer = new byte[6];
@@ -103,7 +101,6 @@ public class HoodieLogFileReader implements HoodieLogFormat.Reader {
public HoodieLogFileReader(HoodieStorage storage, HoodieLogFile logFile, Schema readerSchema, int bufferSize, boolean reverseReader,
boolean enableRecordLookups, String keyField, InternalSchema internalSchema) throws IOException {
this.storage = storage;
- this.storageConf = this.storage.getConf();
// NOTE: We repackage {@code HoodieLogFile} here to make sure that the provided path
// is prefixed with an appropriate scheme given that we're not propagating the FS
// further
@@ -184,7 +181,7 @@ public class HoodieLogFileReader implements HoodieLogFormat.Reader {
long blockEndPos = inputStream.getPos();
HoodieLogBlock.HoodieLogBlockContentLocation logBlockContentLoc =
- new HoodieLogBlock.HoodieLogBlockContentLocation(storageConf, logFile, contentPosition, contentLength, blockEndPos);
+ new HoodieLogBlock.HoodieLogBlockContentLocation(storage, logFile, contentPosition, contentLength, blockEndPos);
switch (Objects.requireNonNull(blockType)) {
case AVRO_DATA_BLOCK:
@@ -259,7 +256,7 @@ public class HoodieLogFileReader implements HoodieLogFormat.Reader {
long contentPosition = inputStream.getPos();
Option<byte[]> corruptedBytes = HoodieLogBlock.tryReadContent(inputStream, corruptedBlockSize, true);
HoodieLogBlock.HoodieLogBlockContentLocation logBlockContentLoc =
- new HoodieLogBlock.HoodieLogBlockContentLocation(storageConf, logFile, contentPosition, corruptedBlockSize, nextBlockOffset);
+ new HoodieLogBlock.HoodieLogBlockContentLocation(storage, logFile, contentPosition, corruptedBlockSize, nextBlockOffset);
return new HoodieCorruptBlock(corruptedBytes, () -> getDataInputStream(storage, this.logFile, bufferSize), true, Option.of(logBlockContentLoc), new HashMap<>(), new HashMap<>());
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java
index 5a8e546734b..b120a364f41 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java
@@ -29,7 +29,7 @@ import org.apache.hudi.common.util.collection.CloseableMappingIterator;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.internal.schema.InternalSchema;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericDatumReader;
@@ -99,7 +99,7 @@ public class HoodieAvroDataBlock extends HoodieDataBlock {
}
@Override
- protected byte[] serializeRecords(List<HoodieRecord> records, StorageConfiguration<?> storageConf) throws IOException {
+ protected byte[] serializeRecords(List<HoodieRecord> records, HoodieStorage storage) throws IOException {
Schema schema = new Schema.Parser().parse(super.getLogBlockHeader().get(HeaderMetadataType.SCHEMA));
GenericDatumWriter<IndexedRecord> writer = new GenericDatumWriter<>(schema);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java
index a519f80eb40..ac0d02e42c2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java
@@ -20,7 +20,7 @@ package org.apache.hudi.common.table.log.block;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import java.util.HashMap;
import java.util.Map;
@@ -62,7 +62,7 @@ public class HoodieCommandBlock extends HoodieLogBlock {
}
@Override
- public byte[] getContentBytes(StorageConfiguration<?> storageConf) {
+ public byte[] getContentBytes(HoodieStorage storage) {
return new byte[0];
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java
index 74502ee1b8b..4de71e7759e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java
@@ -20,7 +20,7 @@ package org.apache.hudi.common.table.log.block;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import java.io.IOException;
import java.util.Map;
@@ -39,7 +39,7 @@ public class HoodieCorruptBlock extends HoodieLogBlock {
}
@Override
- public byte[] getContentBytes(StorageConfiguration<?> storageConf) throws IOException {
+ public byte[] getContentBytes(HoodieStorage storage) throws IOException {
if (!getContent().isPresent() && readBlockLazily) {
// read content from disk
inflate();
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java
index 0b1fcc6dc02..2b35826b1c3 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java
@@ -24,7 +24,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.avro.Schema;
@@ -104,7 +104,7 @@ public abstract class HoodieDataBlock extends HoodieLogBlock {
}
@Override
- public byte[] getContentBytes(StorageConfiguration<?> storageConf) throws IOException {
+ public byte[] getContentBytes(HoodieStorage storage) throws IOException {
// In case this method is called before realizing records from content
Option<byte[]> content = getContent();
@@ -114,7 +114,7 @@ public abstract class HoodieDataBlock extends HoodieLogBlock {
return content.get();
}
- return serializeRecords(records.get(), storageConf);
+ return serializeRecords(records.get(), storage);
}
public String getKeyFieldName() {
@@ -190,7 +190,7 @@ public abstract class HoodieDataBlock extends HoodieLogBlock {
);
}
- protected abstract byte[] serializeRecords(List<HoodieRecord> records, StorageConfiguration<?> storageConf) throws IOException;
+ protected abstract byte[] serializeRecords(List<HoodieRecord> records, HoodieStorage storage) throws IOException;
protected abstract <T> ClosableIterator<HoodieRecord<T>> deserializeRecords(byte[] content, HoodieRecordType type) throws IOException;
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java
index aa4432ab7e4..33b6d0d4cce 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java
@@ -27,7 +27,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.SerializationUtils;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.util.Lazy;
import org.apache.avro.io.BinaryDecoder;
@@ -88,7 +88,7 @@ public class HoodieDeleteBlock extends HoodieLogBlock {
}
@Override
- public byte[] getContentBytes(StorageConfiguration<?> storageConf) throws IOException {
+ public byte[] getContentBytes(HoodieStorage storage) throws IOException {
Option<byte[]> content = getContent();
// In case this method is called before realizing keys from content
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java
index d6fbb52fc7e..e997f5e9aaa 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieHFileDataBlock.java
@@ -33,7 +33,6 @@ import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.inline.InLineFSUtils;
@@ -103,11 +102,11 @@ public class HoodieHFileDataBlock extends HoodieDataBlock {
}
@Override
- protected byte[] serializeRecords(List<HoodieRecord> records, StorageConfiguration<?> storageConf) throws IOException {
+ protected byte[] serializeRecords(List<HoodieRecord> records, HoodieStorage storage) throws IOException {
Schema writerSchema = new Schema.Parser().parse(
super.getLogBlockHeader().get(HoodieLogBlock.HeaderMetadataType.SCHEMA));
return FileFormatUtils.getInstance(HoodieFileFormat.HFILE).serializeRecordsToLogBlock(
- storageConf, records, writerSchema, getSchema(), getKeyFieldName(),
+ storage, records, writerSchema, getSchema(), getKeyFieldName(),
Collections.singletonMap(HFILE_COMPRESSION_ALGORITHM_NAME.key(), compressionCodec.get()));
}
@@ -115,13 +114,13 @@ public class HoodieHFileDataBlock extends HoodieDataBlock {
protected <T> ClosableIterator<HoodieRecord<T>> deserializeRecords(byte[] content, HoodieRecordType type) throws IOException {
checkState(readerSchema != null, "Reader's schema has to be non-null");
- StorageConfiguration<?> storageConf = getBlockContentLocation().get().getStorageConf().getInline();
- HoodieStorage storage = HoodieStorageUtils.getStorage(pathForReader, storageConf);
+ StorageConfiguration<?> storageConf = getBlockContentLocation().get().getStorage().getConf().getInline();
+ HoodieStorage inlineStorage = getBlockContentLocation().get().getStorage().newInstance(pathForReader, storageConf);
// Read the content
- try (HoodieFileReader reader = HoodieIOFactory.getIOFactory(storageConf)
+ try (HoodieFileReader reader = HoodieIOFactory.getIOFactory(inlineStorage)
.getReaderFactory(HoodieRecordType.AVRO)
.getContentReader(hFileReaderConfig, pathForReader, HoodieFileFormat.HFILE,
- storage, content, Option.of(getSchemaFromHeader()))) {
+ inlineStorage, content, Option.of(getSchemaFromHeader()))) {
return unsafeCast(reader.getRecordIterator(readerSchema));
}
}
@@ -133,15 +132,16 @@ public class HoodieHFileDataBlock extends HoodieDataBlock {
// NOTE: It's important to extend Hadoop configuration here to make sure configuration
// is appropriately carried over
- StorageConfiguration<?> inlineConf = blockContentLoc.getStorageConf().getInline();
-
+ StorageConfiguration<?> inlineConf = getBlockContentLocation().get().getStorage().getConf().getInline();
StoragePath inlinePath = InLineFSUtils.getInlineFilePath(
blockContentLoc.getLogFile().getPath(),
blockContentLoc.getLogFile().getPath().toUri().getScheme(),
blockContentLoc.getContentPositionInLogFile(),
blockContentLoc.getBlockSize());
+ HoodieStorage inlineStorage = getBlockContentLocation().get().getStorage().newInstance(inlinePath, inlineConf);
- try (final HoodieAvroHFileReaderImplBase reader = (HoodieAvroHFileReaderImplBase) HoodieIOFactory.getIOFactory(inlineConf)
+ try (final HoodieAvroHFileReaderImplBase reader = (HoodieAvroHFileReaderImplBase) HoodieIOFactory
+ .getIOFactory(inlineStorage)
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(hFileReaderConfig, inlinePath, HoodieFileFormat.HFILE, Option.of(getSchemaFromHeader()))) {
// Get writer's schema from the header
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java
index 70a04d594d1..f98097e39d5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java
@@ -26,7 +26,7 @@ import org.apache.hudi.common.util.TypeUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.SeekableDataInputStream;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.roaringbitmap.longlong.Roaring64NavigableMap;
@@ -85,7 +85,7 @@ public abstract class HoodieLogBlock {
}
// Return the bytes representation of the data belonging to a LogBlock
- public byte[] getContentBytes(StorageConfiguration<?> storageConf) throws IOException {
+ public byte[] getContentBytes(HoodieStorage storage) throws IOException {
throw new HoodieException("No implementation was provided");
}
@@ -183,7 +183,7 @@ public abstract class HoodieLogBlock {
*/
public static final class HoodieLogBlockContentLocation {
// Storage Config required to access the file
- private final StorageConfiguration<?> storageConf;
+ private final HoodieStorage storage;
// The logFile that contains this block
private final HoodieLogFile logFile;
// The filePosition in the logFile for the contents of this block
@@ -193,20 +193,20 @@ public abstract class HoodieLogBlock {
// The final position where the complete block ends
private final long blockEndPos;
- public HoodieLogBlockContentLocation(StorageConfiguration<?> storageConf,
+ public HoodieLogBlockContentLocation(HoodieStorage storage,
HoodieLogFile logFile,
long contentPositionInLogFile,
long blockSize,
long blockEndPos) {
- this.storageConf = storageConf;
+ this.storage = storage;
this.logFile = logFile;
this.contentPositionInLogFile = contentPositionInLogFile;
this.blockSize = blockSize;
this.blockEndPos = blockEndPos;
}
- public StorageConfiguration<?> getStorageConf() {
- return storageConf;
+ public HoodieStorage getStorage() {
+ return storage;
}
public HoodieLogFile getLogFile() {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java
index b94b92a942a..d96941e592f 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieParquetDataBlock.java
@@ -25,6 +25,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.io.SeekableDataInputStream;
import org.apache.hudi.io.storage.HoodieIOFactory;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.inline.InLineFSUtils;
@@ -87,7 +88,7 @@ public class HoodieParquetDataBlock extends HoodieDataBlock {
}
@Override
- protected byte[] serializeRecords(List<HoodieRecord> records, StorageConfiguration<?> storageConf) throws IOException {
+ protected byte[] serializeRecords(List<HoodieRecord> records, HoodieStorage storage) throws IOException {
Map<String, String> paramsMap = new HashMap<>();
paramsMap.put(PARQUET_COMPRESSION_CODEC_NAME.key(), compressionCodecName.get());
paramsMap.put(PARQUET_COMPRESSION_RATIO_FRACTION.key(), String.valueOf(expectedCompressionRatio.get()));
@@ -96,7 +97,7 @@ public class HoodieParquetDataBlock extends HoodieDataBlock {
super.getLogBlockHeader().get(HoodieLogBlock.HeaderMetadataType.SCHEMA));
return FileFormatUtils.getInstance(PARQUET).serializeRecordsToLogBlock(
- storageConf, records, writerSchema, getSchema(), getKeyFieldName(), paramsMap);
+ storage, records, writerSchema, getSchema(), getKeyFieldName(), paramsMap);
}
/**
@@ -110,17 +111,18 @@ public class HoodieParquetDataBlock extends HoodieDataBlock {
// NOTE: It's important to extend Hadoop configuration here to make sure configuration
// is appropriately carried over
- StorageConfiguration<?> inlineConf = blockContentLoc.getStorageConf().getInline();
-
+ StorageConfiguration<?> inlineConf = blockContentLoc.getStorage().getConf().getInline();
StoragePath inlineLogFilePath = InLineFSUtils.getInlineFilePath(
blockContentLoc.getLogFile().getPath(),
blockContentLoc.getLogFile().getPath().toUri().getScheme(),
blockContentLoc.getContentPositionInLogFile(),
blockContentLoc.getBlockSize());
+ HoodieStorage inlineStorage = getBlockContentLocation().get().getStorage().newInstance(inlineLogFilePath, inlineConf);
Schema writerSchema = new Schema.Parser().parse(this.getLogBlockHeader().get(HeaderMetadataType.SCHEMA));
- ClosableIterator<HoodieRecord<T>> iterator = HoodieIOFactory.getIOFactory(inlineConf).getReaderFactory(type)
+ ClosableIterator<HoodieRecord<T>> iterator = HoodieIOFactory.getIOFactory(inlineStorage)
+ .getReaderFactory(type)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, inlineLogFilePath, PARQUET, Option.empty())
.getRecordIterator(writerSchema, readerSchema);
return iterator;
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
index 7f53feb5a54..e1892798ee0 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
@@ -30,7 +30,6 @@ import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
import org.slf4j.Logger;
@@ -815,10 +814,9 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
StoragePath srcPath = new StoragePath(metaClient.getMetaPath(), instant.getFileName());
StoragePath dstPath = new StoragePath(dstDir, instant.getFileName());
try {
- HoodieStorage srcStorage = HoodieStorageUtils.getStorage(srcPath, metaClient.getStorageConf());
- HoodieStorage dstStorage = HoodieStorageUtils.getStorage(dstPath, metaClient.getStorageConf());
- dstStorage.createDirectory(dstDir);
- FileIOUtils.copy(srcStorage, srcPath, dstStorage, dstPath, false, true);
+ HoodieStorage storage = metaClient.getStorage();
+ storage.createDirectory(dstDir);
+ FileIOUtils.copy(storage, srcPath, storage, dstPath, false, true);
} catch (IOException e) {
throw new HoodieIOException("Could not copy instant from " + srcPath + " to " + dstPath, e);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
index 30eefc92907..d34952aa0c8 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
@@ -220,7 +220,7 @@ public class FileSystemViewManager {
final FileSystemViewStorageConfig config,
final HoodieCommonConfig commonConfig) {
return createViewManager(context, config, commonConfig,
- metaClient -> HoodieTableMetadata.create(context, metadataConfig, metaClient.getBasePathV2().toString(), true));
+ metaClient -> HoodieTableMetadata.create(context, metaClient.getStorage(), metadataConfig, metaClient.getBasePathV2().toString(), true));
}
public static FileSystemViewManager createViewManager(final HoodieEngineContext context,
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java
index d5620fdcf65..e12b5a05ec8 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java
@@ -31,7 +31,6 @@ import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -81,24 +80,24 @@ public abstract class FileFormatUtils {
/**
* Read the rowKey list from the given data file.
*
- * @param configuration configuration to build storage object.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return set of row keys
*/
- public Set<String> readRowKeys(StorageConfiguration<?> configuration, StoragePath filePath) {
- return filterRowKeys(configuration, filePath, new HashSet<>());
+ public Set<String> readRowKeys(HoodieStorage storage, StoragePath filePath) {
+ return filterRowKeys(storage, filePath, new HashSet<>());
}
/**
* Read the bloom filter from the metadata of the given data file.
*
- * @param configuration configuration.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return a BloomFilter object.
*/
- public BloomFilter readBloomFilterFromMetadata(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public BloomFilter readBloomFilterFromMetadata(HoodieStorage storage, StoragePath filePath) {
Map<String, String> footerVals =
- readFooter(configuration, false, filePath,
+ readFooter(storage, false, filePath,
HoodieBloomFilterWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY,
HoodieBloomFilterWriteSupport.OLD_HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY,
HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE);
@@ -122,12 +121,12 @@ public abstract class FileFormatUtils {
/**
* Read the min and max record key from the metadata of the given data file.
*
- * @param configuration configuration.
- * @param filePath the data file path.
- * @return a array of two string where the first is min record key and the second is max record key.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
+ * @return an array of two string where the first is min record key and the second is max record key.
*/
- public String[] readMinMaxRecordKeys(StorageConfiguration<?> configuration, StoragePath filePath) {
- Map<String, String> minMaxKeys = readFooter(configuration, true, filePath,
+ public String[] readMinMaxRecordKeys(HoodieStorage storage, StoragePath filePath) {
+ Map<String, String> minMaxKeys = readFooter(storage, true, filePath,
HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER, HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER);
if (minMaxKeys.size() != 2) {
throw new HoodieException(
@@ -142,114 +141,114 @@ public abstract class FileFormatUtils {
* Read the data file
* NOTE: This literally reads the entire file contents, thus should be used with caution.
*
- * @param configuration configuration.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return a list of GenericRecord.
*/
- public abstract List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath);
+ public abstract List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath);
/**
* Read the data file using the given schema
* NOTE: This literally reads the entire file contents, thus should be used with caution.
*
- * @param configuration configuration.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return a list of GenericRecord.
*/
- public abstract List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath, Schema schema);
+ public abstract List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath, Schema schema);
/**
* Read the footer data of the given data file.
*
- * @param configuration configuration.
- * @param required require the footer data to be in data file.
- * @param filePath the data file path.
- * @param footerNames the footer names to read.
+ * @param storage {@link HoodieStorage} instance.
+ * @param required require the footer data to be in data file.
+ * @param filePath the data file path.
+ * @param footerNames the footer names to read.
* @return a map where the key is the footer name and the value is the footer value.
*/
- public abstract Map<String, String> readFooter(StorageConfiguration<?> configuration, boolean required, StoragePath filePath,
+ public abstract Map<String, String> readFooter(HoodieStorage storage, boolean required, StoragePath filePath,
String... footerNames);
/**
* Returns the number of records in the data file.
*
- * @param configuration configuration.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
*/
- public abstract long getRowCount(StorageConfiguration<?> configuration, StoragePath filePath);
+ public abstract long getRowCount(HoodieStorage storage, StoragePath filePath);
/**
* Read the rowKey list matching the given filter, from the given data file.
* If the filter is empty, then this will return all the row keys.
*
- * @param configuration configuration to build storage object.
- * @param filePath the data file path.
- * @param filter record keys filter.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
+ * @param filter record keys filter.
* @return set of row keys matching candidateRecordKeys.
*/
- public abstract Set<String> filterRowKeys(StorageConfiguration<?> configuration, StoragePath filePath, Set<String> filter);
+ public abstract Set<String> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set<String> filter);
/**
* Fetch {@link HoodieKey}s from the given data file.
*
- * @param configuration configuration to build storage object.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return {@link List} of {@link HoodieKey}s fetched from the data file.
*/
- public abstract List<HoodieKey> fetchHoodieKeys(StorageConfiguration<?> configuration, StoragePath filePath);
+ public abstract List<HoodieKey> fetchHoodieKeys(HoodieStorage storage, StoragePath filePath);
/**
* Provides a closable iterator for reading the given data file.
*
- * @param configuration configuration to build storage object.
+ * @param storage {@link HoodieStorage} instance.
* @param filePath the data file path.
* @param keyGeneratorOpt instance of KeyGenerator.
* @return {@link ClosableIterator} of {@link HoodieKey}s for reading the file.
*/
- public abstract ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration,
+ public abstract ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage,
StoragePath filePath,
Option<BaseKeyGenerator> keyGeneratorOpt);
/**
* Provides a closable iterator for reading the given data file.
*
- * @param configuration configuration to build storage object.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return {@link ClosableIterator} of {@link HoodieKey}s for reading the file.
*/
- public abstract ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath);
+ public abstract ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath);
/**
* Fetch {@link HoodieKey}s from the given data file.
*
- * @param configuration configuration to build storage object.
+ * @param storage {@link HoodieStorage} instance.
* @param filePath the data file path.
* @param keyGeneratorOpt instance of KeyGenerator.
* @return {@link List} of{@link HoodieKey}s fetched from the data file.
*/
- public abstract List<HoodieKey> fetchHoodieKeys(StorageConfiguration<?> configuration,
+ public abstract List<HoodieKey> fetchHoodieKeys(HoodieStorage storage,
StoragePath filePath,
Option<BaseKeyGenerator> keyGeneratorOpt);
/**
* Read the Avro schema of the data file.
*
- * @param configuration configuration.
- * @param filePath the data file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
* @return the Avro schema of the data file.
*/
- public abstract Schema readAvroSchema(StorageConfiguration<?> configuration, StoragePath filePath);
+ public abstract Schema readAvroSchema(HoodieStorage storage, StoragePath filePath);
/**
* Reads column statistics stored in the metadata.
*
- * @param storageConf storage configuration.
- * @param filePath the data file path.
- * @param columnList List of columns to get column statistics.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath the data file path.
+ * @param columnList List of columns to get column statistics.
* @return {@link List} of {@link HoodieColumnRangeMetadata}.
*/
@SuppressWarnings("rawtype")
- public abstract List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(StorageConfiguration<?> storageConf,
+ public abstract List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(HoodieStorage storage,
StoragePath filePath,
List<String> columnList);
@@ -273,7 +272,7 @@ public abstract class FileFormatUtils {
/**
* Serializes Hudi records to the log block.
*
- * @param storageConf storage configuration.
+ * @param storage {@link HoodieStorage} instance.
* @param records a list of {@link HoodieRecord}.
* @param writerSchema writer schema string from the log block header.
* @param readerSchema
@@ -282,7 +281,7 @@ public abstract class FileFormatUtils {
* @return byte array after serialization.
* @throws IOException upon serialization error.
*/
- public abstract byte[] serializeRecordsToLogBlock(StorageConfiguration<?> storageConf,
+ public abstract byte[] serializeRecordsToLogBlock(HoodieStorage storage,
List<HoodieRecord> records,
Schema writerSchema,
Schema readerSchema, String keyFieldName,
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java b/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java
index 407cd7103e3..8e1451d5257 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java
@@ -31,8 +31,6 @@ import org.apache.hudi.internal.schema.io.FileBasedInternalSchemaStorageManager;
import org.apache.hudi.internal.schema.utils.InternalSchemaUtils;
import org.apache.hudi.internal.schema.utils.SerDeHelper;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import com.github.benmanes.caffeine.cache.Cache;
@@ -174,19 +172,19 @@ public class InternalSchemaCache {
* step3:
* if we cannot parser internalSchema in step2 (eg: schema evolution is not enabled when we create hoodie table, however after some inserts we enable schema evolution)
* try to convert table schema to internalSchema.
- * @param versionId the internalSchema version to be search.
- * @param tablePath table path
- * @param storageConf conf
+ *
+ * @param versionId the internalSchema version to be search.
+ * @param tablePath table path
+ * @param storage {@link HoodieStorage} instance.
* @param validCommits current validate commits, use to make up the commit file path/verify the validity of the history schema files
* @return a internalSchema.
*/
- public static InternalSchema getInternalSchemaByVersionId(long versionId, String tablePath, StorageConfiguration<?> storageConf, String validCommits) {
+ public static InternalSchema getInternalSchemaByVersionId(long versionId, String tablePath, HoodieStorage storage, String validCommits) {
String avroSchema = "";
Set<String> commitSet = Arrays.stream(validCommits.split(",")).collect(Collectors.toSet());
List<String> validateCommitList =
commitSet.stream().map(HoodieInstant::extractTimestamp).collect(Collectors.toList());
- HoodieStorage storage = HoodieStorageUtils.getStorage(tablePath, storageConf);
StoragePath hoodieMetaPath = new StoragePath(tablePath, HoodieTableMetaClient.METAFOLDER_NAME);
//step1:
StoragePath candidateCommitFile = commitSet.stream()
@@ -215,7 +213,7 @@ public class InternalSchemaCache {
}
// step2:
FileBasedInternalSchemaStorageManager fileBasedInternalSchemaStorageManager =
- new FileBasedInternalSchemaStorageManager(storageConf, new StoragePath(tablePath));
+ new FileBasedInternalSchemaStorageManager(storage, new StoragePath(tablePath));
String latestHistorySchema =
fileBasedInternalSchemaStorageManager.getHistorySchemaStrByGivenValidCommits(validateCommitList);
if (latestHistorySchema.isEmpty()) {
@@ -234,7 +232,7 @@ public class InternalSchemaCache {
public static InternalSchema getInternalSchemaByVersionId(long versionId, HoodieTableMetaClient metaClient) {
String validCommitLists = metaClient
.getCommitsAndCompactionTimeline().filterCompletedInstants().getInstantsAsStream().map(HoodieInstant::getFileName).collect(Collectors.joining(","));
- return getInternalSchemaByVersionId(versionId, metaClient.getBasePathV2().toString(), metaClient.getStorageConf(), validCommitLists);
+ return getInternalSchemaByVersionId(versionId, metaClient.getBasePathV2().toString(), metaClient.getStorage(), validCommitLists);
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
index 43923b5e40a..5737e2dcec0 100644
--- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
+++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
@@ -29,8 +29,6 @@ import org.apache.hudi.internal.schema.InternalSchema;
import org.apache.hudi.internal.schema.utils.InternalSchemaUtils;
import org.apache.hudi.internal.schema.utils.SerDeHelper;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.slf4j.Logger;
@@ -55,25 +53,25 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem
public static final String SCHEMA_NAME = ".schema";
private final StoragePath baseSchemaPath;
- private final StorageConfiguration<?> conf;
+ private final HoodieStorage storage;
private HoodieTableMetaClient metaClient;
- public FileBasedInternalSchemaStorageManager(StorageConfiguration<?> conf, StoragePath baseTablePath) {
+ public FileBasedInternalSchemaStorageManager(HoodieStorage storage, StoragePath baseTablePath) {
StoragePath metaPath = new StoragePath(baseTablePath, HoodieTableMetaClient.METAFOLDER_NAME);
this.baseSchemaPath = new StoragePath(metaPath, SCHEMA_NAME);
- this.conf = conf;
+ this.storage = storage;
}
public FileBasedInternalSchemaStorageManager(HoodieTableMetaClient metaClient) {
this.baseSchemaPath = new StoragePath(metaClient.getMetaPath(), SCHEMA_NAME);
- this.conf = metaClient.getStorageConf();
+ this.storage = metaClient.getStorage();
this.metaClient = metaClient;
}
// make metaClient build lazy
private HoodieTableMetaClient getMetaClient() {
if (metaClient == null) {
- metaClient = HoodieTableMetaClient.builder().setBasePath(baseSchemaPath.getParent().getParent().toString()).setConf(conf.newInstance()).build();
+ metaClient = HoodieTableMetaClient.builder().setBasePath(baseSchemaPath.getParent().getParent().toString()).setConf(storage.getConf().newInstance()).build();
}
return metaClient;
}
@@ -93,7 +91,6 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem
private void cleanResidualFiles() {
List<String> validateCommits = getValidInstants();
try {
- HoodieStorage storage = HoodieStorageUtils.getStorage(baseSchemaPath, conf);
if (storage.exists(baseSchemaPath)) {
List<String> candidateSchemaFiles = storage.listDirectEntries(baseSchemaPath).stream()
.filter(f -> f.isFile())
@@ -117,7 +114,6 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem
public void cleanOldFiles(List<String> validateCommits) {
try {
- HoodieStorage storage = HoodieStorageUtils.getStorage(baseSchemaPath, conf);
if (storage.exists(baseSchemaPath)) {
List<String> candidateSchemaFiles = storage.listDirectEntries(baseSchemaPath).stream()
.filter(f -> f.isFile())
@@ -148,7 +144,6 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem
public String getHistorySchemaStrByGivenValidCommits(List<String> validCommits) {
List<String> commitList = validCommits == null || validCommits.isEmpty() ? getValidInstants() : validCommits;
try {
- HoodieStorage storage = HoodieStorageUtils.getStorage(baseSchemaPath, conf);
if (storage.exists(baseSchemaPath)) {
List<String> validaSchemaFiles = storage.listDirectEntries(baseSchemaPath).stream()
.filter(f -> f.isFile() && f.getPath().getName().endsWith(SCHEMA_COMMIT_ACTION))
diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java
index 18dd976798d..0c63a3057c2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java
+++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileReaderFactory.java
@@ -24,7 +24,6 @@ import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -40,9 +39,9 @@ import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET;
*/
public class HoodieFileReaderFactory {
- protected final StorageConfiguration<?> storageConf;
- public HoodieFileReaderFactory(StorageConfiguration<?> storageConf) {
- this.storageConf = storageConf;
+ protected final HoodieStorage storage;
+ public HoodieFileReaderFactory(HoodieStorage storage) {
+ this.storage = storage;
}
public HoodieFileReader getFileReader(HoodieConfig hoodieConfig, StoragePath path) throws IOException {
diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java
index 65b172136c1..72cfff6d622 100644
--- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java
+++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java
@@ -26,7 +26,7 @@ import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -39,24 +39,24 @@ import static org.apache.hudi.common.model.HoodieFileFormat.ORC;
import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET;
public class HoodieFileWriterFactory {
- protected final StorageConfiguration<?> storageConf;
+ protected final HoodieStorage storage;
- public HoodieFileWriterFactory(StorageConfiguration<?> storageConf) {
- this.storageConf = storageConf;
+ public HoodieFileWriterFactory(HoodieStorage storage) {
+ this.storage = storage;
}
public static <T, I, K, O> HoodieFileWriter getFileWriter(
- String instantTime, StoragePath path, StorageConfiguration<?> conf, HoodieConfig config, Schema schema,
+ String instantTime, StoragePath path, HoodieStorage storage, HoodieConfig config, Schema schema,
TaskContextSupplier taskContextSupplier, HoodieRecordType recordType) throws IOException {
final String extension = FSUtils.getFileExtension(path.getName());
- HoodieFileWriterFactory factory = HoodieIOFactory.getIOFactory(conf).getWriterFactory(recordType);
+ HoodieFileWriterFactory factory = HoodieIOFactory.getIOFactory(storage).getWriterFactory(recordType);
return factory.getFileWriterByFormat(extension, instantTime, path, config, schema, taskContextSupplier);
}
public static <T, I, K, O> HoodieFileWriter getFileWriter(HoodieFileFormat format, OutputStream outputStream,
- StorageConfiguration<?> conf, HoodieConfig config, Schema schema, HoodieRecordType recordType)
+ HoodieStorage storage, HoodieConfig config, Schema schema, HoodieRecordType recordType)
throws IOException {
- HoodieFileWriterFactory factory = HoodieIOFactory.getIOFactory(conf).getWriterFactory(recordType);
+ HoodieFileWriterFactory factory = HoodieIOFactory.getIOFactory(storage).getWriterFactory(recordType);
return factory.getFileWriterByFormat(format, outputStream, config, schema);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieIOFactory.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieIOFactory.java
index 3ae6b60321e..cba3c7b0e98 100644
--- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieIOFactory.java
+++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieIOFactory.java
@@ -25,25 +25,24 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
/**
* Base class to get HoodieFileReaderFactory and HoodieFileWriterFactory
*/
public abstract class HoodieIOFactory {
- protected final StorageConfiguration<?> storageConf;
+ protected final HoodieStorage storage;
- public HoodieIOFactory(StorageConfiguration<?> storageConf) {
- this.storageConf = storageConf;
+ public HoodieIOFactory(HoodieStorage storage) {
+ this.storage = storage;
}
- public static HoodieIOFactory getIOFactory(StorageConfiguration<?> storageConf) {
- String ioFactoryClass = storageConf.getString(HoodieStorageConfig.HOODIE_IO_FACTORY_CLASS.key())
+ public static HoodieIOFactory getIOFactory(HoodieStorage storage) {
+ String ioFactoryClass = storage.getConf().getString(HoodieStorageConfig.HOODIE_IO_FACTORY_CLASS.key())
.orElse(HoodieStorageConfig.HOODIE_IO_FACTORY_CLASS.defaultValue());
try {
return (HoodieIOFactory) ReflectionUtils
- .loadClass(ioFactoryClass, new Class<?>[] {StorageConfiguration.class}, storageConf);
+ .loadClass(ioFactoryClass, new Class<?>[] {HoodieStorage.class}, storage);
} catch (Exception e) {
throw new HoodieException("Unable to create " + ioFactoryClass, e);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java
index e47e393e6a6..b769c2e88a0 100644
--- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java
+++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java
@@ -36,8 +36,6 @@ import org.apache.hudi.io.hfile.HFileReaderImpl;
import org.apache.hudi.io.hfile.KeyValue;
import org.apache.hudi.io.hfile.UTF8StringKey;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.util.Lazy;
@@ -67,14 +65,14 @@ import static org.apache.hudi.io.hfile.HFileUtils.isPrefixOfKey;
public class HoodieNativeAvroHFileReader extends HoodieAvroHFileReaderImplBase {
private static final Logger LOG = LoggerFactory.getLogger(HoodieNativeAvroHFileReader.class);
- private final StorageConfiguration<?> conf;
+ private final HoodieStorage storage;
private final Option<StoragePath> path;
private final Option<byte[]> bytesContent;
private Option<HFileReader> sharedHFileReader;
private final Lazy<Schema> schema;
- public HoodieNativeAvroHFileReader(StorageConfiguration<?> conf, StoragePath path, Option<Schema> schemaOption) {
- this.conf = conf;
+ public HoodieNativeAvroHFileReader(HoodieStorage storage, StoragePath path, Option<Schema> schemaOption) {
+ this.storage = storage;
this.path = Option.of(path);
this.bytesContent = Option.empty();
this.sharedHFileReader = Option.empty();
@@ -82,8 +80,8 @@ public class HoodieNativeAvroHFileReader extends HoodieAvroHFileReaderImplBase {
.orElseGet(() -> Lazy.lazily(() -> fetchSchema(getSharedHFileReader())));
}
- public HoodieNativeAvroHFileReader(StorageConfiguration<?> conf, byte[] content, Option<Schema> schemaOption) {
- this.conf = conf;
+ public HoodieNativeAvroHFileReader(HoodieStorage storage, byte[] content, Option<Schema> schemaOption) {
+ this.storage = storage;
this.path = Option.empty();
this.bytesContent = Option.of(content);
this.sharedHFileReader = Option.empty();
@@ -257,7 +255,6 @@ public class HoodieNativeAvroHFileReader extends HoodieAvroHFileReaderImplBase {
SeekableDataInputStream inputStream;
long fileSize;
if (path.isPresent()) {
- HoodieStorage storage = HoodieStorageUtils.getStorage(path.get(), conf);
fileSize = storage.getPathInfo(path.get()).getLength();
inputStream = storage.openSeekable(path.get(), false);
} else {
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/AbstractHoodieTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/AbstractHoodieTableMetadata.java
index 9128b82a3c5..ea61cd1130e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/AbstractHoodieTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/AbstractHoodieTableMetadata.java
@@ -24,6 +24,7 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.expression.ArrayData;
import org.apache.hudi.internal.schema.Type;
import org.apache.hudi.internal.schema.Types;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
@@ -35,6 +36,7 @@ import java.util.stream.IntStream;
public abstract class AbstractHoodieTableMetadata implements HoodieTableMetadata {
protected transient HoodieEngineContext engineContext;
+ protected transient HoodieStorage storage;
protected final StorageConfiguration<?> storageConf;
protected final StoragePath dataBasePath;
@@ -42,9 +44,10 @@ public abstract class AbstractHoodieTableMetadata implements HoodieTableMetadata
// TODO get this from HoodieConfig
protected final boolean caseSensitive = false;
- public AbstractHoodieTableMetadata(HoodieEngineContext engineContext, StorageConfiguration<?> conf, String dataBasePath) {
+ public AbstractHoodieTableMetadata(HoodieEngineContext engineContext, HoodieStorage storage, String dataBasePath) {
this.engineContext = engineContext;
- this.storageConf = conf;
+ this.storage = storage;
+ this.storageConf = storage.getConf();
this.dataBasePath = new StoragePath(dataBasePath);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
index f9e8bf2b7c4..c3bd5c636c0 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
@@ -39,10 +39,9 @@ import org.apache.hudi.common.util.hash.ColumnIndexID;
import org.apache.hudi.common.util.hash.FileIndexID;
import org.apache.hudi.common.util.hash.PartitionIndexID;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -84,11 +83,14 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
protected final boolean hiveStylePartitioningEnabled;
protected final boolean urlEncodePartitioningEnabled;
- protected BaseTableMetadata(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig, String dataBasePath) {
- super(engineContext, engineContext.getStorageConf(), dataBasePath);
+ protected BaseTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig,
+ String dataBasePath) {
+ super(engineContext, storage, dataBasePath);
this.dataMetaClient = HoodieTableMetaClient.builder()
- .setConf(storageConf.newInstance())
+ .setConf(storage.getConf().newInstance())
.setBasePath(dataBasePath)
.build();
@@ -99,7 +101,7 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
if (metadataConfig.isMetricsEnabled()) {
this.metrics = Option.of(new HoodieMetadataMetrics(HoodieMetricsConfig.newBuilder()
- .fromProperties(metadataConfig.getProps()).build(), getStorageConf()));
+ .fromProperties(metadataConfig.getProps()).build(), dataMetaClient.getStorage()));
} else {
this.metrics = Option.empty();
}
@@ -359,9 +361,9 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
HoodieMetadataPayload metadataPayload = record.getData();
checkForSpuriousDeletes(metadataPayload, recordKey);
try {
- return metadataPayload.getFileList(getStorageConf(), partitionPath);
- } catch (IOException e) {
- throw new HoodieIOException("Failed to extract file-pathInfoList from the payload", e);
+ return metadataPayload.getFileList(dataMetaClient.getStorage(), partitionPath);
+ } catch (Exception e) {
+ throw new HoodieException("Failed to extract file-pathInfoList from the payload", e);
}
})
.orElseGet(Collections::emptyList);
@@ -389,9 +391,6 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
metrics.ifPresent(
m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_FILES_STR, timer.endTimer()));
- HoodieStorage storage =
- HoodieStorageUtils.getStorage(partitionPaths.get(0), getStorageConf());
-
Map<String, List<StoragePathInfo>> partitionPathToFilesMap =
partitionIdRecordPairs.entrySet().stream()
.map(e -> {
@@ -401,7 +400,7 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
HoodieMetadataPayload metadataPayload = e.getValue().getData();
checkForSpuriousDeletes(metadataPayload, partitionId);
- List<StoragePathInfo> files = metadataPayload.getFileList(storage, partitionPath);
+ List<StoragePathInfo> files = metadataPayload.getFileList(dataMetaClient.getStorage(), partitionPath);
return Pair.of(partitionPath.toString(), files);
})
.collect(Collectors.toMap(Pair::getKey, Pair::getValue));
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
index 1148503c5a8..82c56f67c96 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
@@ -41,7 +41,6 @@ import org.apache.hudi.expression.Predicates;
import org.apache.hudi.internal.schema.Types;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -67,9 +66,9 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
private final boolean urlEncodePartitioningEnabled;
public FileSystemBackedTableMetadata(HoodieEngineContext engineContext, HoodieTableConfig tableConfig,
- StorageConfiguration<?> conf, String datasetBasePath,
+ HoodieStorage storage, String datasetBasePath,
boolean assumeDatePartitioning) {
- super(engineContext, conf, datasetBasePath);
+ super(engineContext, storage, datasetBasePath);
this.hiveStylePartitioningEnabled = Boolean.parseBoolean(tableConfig.getHiveStylePartitioningEnable());
this.urlEncodePartitioningEnabled = Boolean.parseBoolean(tableConfig.getUrlEncodePartitioning());
@@ -77,11 +76,10 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
}
public FileSystemBackedTableMetadata(HoodieEngineContext engineContext,
- StorageConfiguration<?> conf, String datasetBasePath,
+ HoodieStorage storage, String datasetBasePath,
boolean assumeDatePartitioning) {
- super(engineContext, conf, datasetBasePath);
+ super(engineContext, storage, datasetBasePath);
- HoodieStorage storage = HoodieStorageUtils.getStorage(dataBasePath, conf);
StoragePath metaPath =
new StoragePath(dataBasePath, HoodieTableMetaClient.METAFOLDER_NAME);
TableNotFoundException.checkTableValidity(storage, this.dataBasePath, metaPath);
@@ -93,10 +91,16 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
this.assumeDatePartitioning = assumeDatePartitioning;
}
+ public HoodieStorage getStorage() {
+ if (storage == null) {
+ storage = HoodieStorageUtils.getStorage(dataBasePath, storageConf);
+ }
+ return storage;
+ }
+
@Override
public List<StoragePathInfo> getAllFilesInPartition(StoragePath partitionPath) throws IOException {
- HoodieStorage storage = HoodieStorageUtils.getStorage(partitionPath, storageConf);
- return FSUtils.getAllDataFilesInPartition(storage, partitionPath);
+ return FSUtils.getAllDataFilesInPartition(getStorage(), partitionPath);
}
@Override
@@ -173,8 +177,7 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
"Listing all partitions with prefix " + relativePathPrefix);
// Need to use serializable file status here, see HUDI-5936
List<StoragePathInfo> dirToFileListing = engineContext.flatMap(pathsToList, path -> {
- HoodieStorage storage = HoodieStorageUtils.getStorage(path, storageConf);
- return storage.listDirectEntries(path).stream();
+ return getStorage().listDirectEntries(path).stream();
}, listingParallelism);
pathsToList.clear();
@@ -189,9 +192,8 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
engineContext.map(dirToFileListing,
fileInfo -> {
StoragePath path = fileInfo.getPath();
- HoodieStorage storage = HoodieStorageUtils.getStorage(path, storageConf);
if (fileInfo.isDirectory()) {
- if (HoodiePartitionMetadata.hasPartitionMetadata(storage, path)) {
+ if (HoodiePartitionMetadata.hasPartitionMetadata(getStorage(), path)) {
return Pair.of(
Option.of(FSUtils.getRelativePartitionPath(dataBasePath,
path)),
@@ -260,9 +262,8 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
engineContext.map(new ArrayList<>(partitionPaths),
partitionPathStr -> {
StoragePath partitionPath = new StoragePath(partitionPathStr);
- HoodieStorage storage = HoodieStorageUtils.getStorage(partitionPath, storageConf);
return Pair.of(partitionPathStr,
- FSUtils.getAllDataFilesInPartition(storage, partitionPath));
+ FSUtils.getAllDataFilesInPartition(getStorage(), partitionPath));
}, parallelism);
return partitionToFiles.stream().collect(Collectors.toMap(pair -> pair.getLeft(),
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
index 2cb42af683b..185791bbbec 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
@@ -51,6 +51,7 @@ import org.apache.hudi.expression.Expression;
import org.apache.hudi.internal.schema.Types;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.io.storage.HoodieSeekingFileReader;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.util.Transient;
@@ -102,13 +103,18 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
// Latest file slices in the metadata partitions
private final Map<String, List<FileSlice>> partitionFileSliceMap = new ConcurrentHashMap<>();
- public HoodieBackedTableMetadata(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig, String datasetBasePath) {
- this(engineContext, metadataConfig, datasetBasePath, false);
+ public HoodieBackedTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig,
+ String datasetBasePath) {
+ this(engineContext, storage, metadataConfig, datasetBasePath, false);
}
- public HoodieBackedTableMetadata(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig,
+ public HoodieBackedTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig,
String datasetBasePath, boolean reuse) {
- super(engineContext, metadataConfig, datasetBasePath);
+ super(engineContext, storage, metadataConfig, datasetBasePath);
this.reuse = reuse;
this.metadataBasePath = HoodieTableMetadata.getMetadataTableBasePath(dataBasePath.toString());
@@ -446,7 +452,8 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
Option<HoodieBaseFile> basefile = slice.getBaseFile();
if (basefile.isPresent()) {
StoragePath baseFilePath = basefile.get().getStoragePath();
- baseFileReader = (HoodieSeekingFileReader<?>) HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecordType.AVRO)
+ baseFileReader = (HoodieSeekingFileReader<?>) HoodieIOFactory.getIOFactory(metadataMetaClient.getStorage())
+ .getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, baseFilePath);
baseFileOpenMs = timer.endTimer();
LOG.info(String.format("Opened metadata base file from %s at instant %s in %d ms", baseFilePath,
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
index 05b3efa9104..624d6099833 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
@@ -25,8 +25,8 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import java.io.IOException;
import java.util.List;
@@ -52,7 +52,8 @@ public class HoodieMetadataFileSystemView extends HoodieTableFileSystemView {
HoodieTableMetaClient metaClient,
HoodieTimeline visibleActiveTimeline,
HoodieMetadataConfig metadataConfig) {
- this(metaClient, visibleActiveTimeline, HoodieTableMetadata.create(engineContext, metadataConfig, metaClient.getBasePath(), true));
+ this(metaClient, visibleActiveTimeline, HoodieTableMetadata.create(
+ engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePath(), true));
}
/**
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
index fce32753883..25a79cb12d8 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
@@ -27,7 +27,7 @@ import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.metrics.HoodieGauge;
import org.apache.hudi.metrics.Metrics;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import com.codahale.metrics.MetricRegistry;
import org.slf4j.Logger;
@@ -81,8 +81,8 @@ public class HoodieMetadataMetrics implements Serializable {
private final transient MetricRegistry metricsRegistry;
private final transient Metrics metrics;
- public HoodieMetadataMetrics(HoodieMetricsConfig metricsConfig, StorageConfiguration<?> storageConf) {
- this.metrics = Metrics.getInstance(metricsConfig, storageConf);
+ public HoodieMetadataMetrics(HoodieMetricsConfig metricsConfig, HoodieStorage storage) {
+ this.metrics = Metrics.getInstance(metricsConfig, storage);
this.metricsRegistry = metrics.getRegistry();
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
index 31c80c5070b..047f384bb88 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataPayload.java
@@ -39,8 +39,6 @@ import org.apache.hudi.common.util.hash.PartitionIndexID;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.util.Lazy;
@@ -497,15 +495,6 @@ public class HoodieMetadataPayload implements HoodieRecordPayload<HoodieMetadata
return Option.of(columnStatMetadata);
}
- /**
- * Returns the files added as part of this record.
- */
- public List<StoragePathInfo> getFileList(StorageConfiguration<?> storageConf, StoragePath partitionPath)
- throws IOException {
- HoodieStorage storage = HoodieStorageUtils.getStorage(partitionPath, storageConf);
- return getFileList(storage, partitionPath);
- }
-
/**
* Returns the files added as part of this record.
*/
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
index d9483eebc64..e36bec14f7d 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
@@ -31,6 +31,7 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.expression.Expression;
import org.apache.hudi.internal.schema.Types;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -111,34 +112,43 @@ public interface HoodieTableMetadata extends Serializable, AutoCloseable {
return basePath.endsWith(HoodieTableMetaClient.METADATA_TABLE_FOLDER_PATH);
}
- static HoodieTableMetadata create(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig, String datasetBasePath) {
- return create(engineContext, metadataConfig, datasetBasePath, false);
+ static HoodieTableMetadata create(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig,
+ String datasetBasePath) {
+ return create(engineContext, storage, metadataConfig, datasetBasePath, false);
}
- static HoodieTableMetadata create(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig, String datasetBasePath, boolean reuse) {
+ static HoodieTableMetadata create(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig,
+ String datasetBasePath,
+ boolean reuse) {
if (metadataConfig.isEnabled()) {
- HoodieBackedTableMetadata metadata = createHoodieBackedTableMetadata(engineContext, metadataConfig, datasetBasePath, reuse);
+ HoodieBackedTableMetadata metadata = createHoodieBackedTableMetadata(engineContext, storage, metadataConfig, datasetBasePath, reuse);
// If the MDT is not initialized then we fallback to FSBackedTableMetadata
if (metadata.isMetadataTableInitialized()) {
return metadata;
}
}
- return createFSBackedTableMetadata(engineContext, metadataConfig, datasetBasePath);
+ return createFSBackedTableMetadata(engineContext, storage, metadataConfig, datasetBasePath);
}
static FileSystemBackedTableMetadata createFSBackedTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
HoodieMetadataConfig metadataConfig,
String datasetBasePath) {
- return new FileSystemBackedTableMetadata(engineContext, engineContext.getStorageConf(),
+ return new FileSystemBackedTableMetadata(engineContext, storage,
datasetBasePath, metadataConfig.shouldAssumeDatePartitioning());
}
static HoodieBackedTableMetadata createHoodieBackedTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
HoodieMetadataConfig metadataConfig,
String datasetBasePath,
boolean reuse) {
- return new HoodieBackedTableMetadata(engineContext, metadataConfig, datasetBasePath, reuse);
+ return new HoodieBackedTableMetadata(engineContext, storage, metadataConfig, datasetBasePath, reuse);
}
/**
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
index edf0d1bc33d..7406943eb47 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
@@ -504,7 +504,7 @@ public class HoodieTableMetadataUtil {
}
final StoragePath writeFilePath = new StoragePath(dataMetaClient.getBasePathV2(), pathWithPartition);
- try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(dataMetaClient.getStorageConf())
+ try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(dataMetaClient.getStorage())
.getReaderFactory(HoodieRecordType.AVRO).getFileReader(hoodieConfig, writeFilePath)) {
try {
final BloomFilter fileBloomFilter = fileReader.readBloomFilter();
@@ -868,7 +868,7 @@ public class HoodieTableMetadataUtil {
if (!isDeleted) {
final String pathWithPartition = partitionName + "/" + filename;
final StoragePath addedFilePath = new StoragePath(dataMetaClient.getBasePathV2(), pathWithPartition);
- bloomFilterBuffer = readBloomFilter(dataMetaClient.getStorageConf(), addedFilePath);
+ bloomFilterBuffer = readBloomFilter(dataMetaClient.getStorage(), addedFilePath);
// If reading the bloom filter failed then do not add a record for this file
if (bloomFilterBuffer == null) {
@@ -923,9 +923,9 @@ public class HoodieTableMetadataUtil {
});
}
- private static ByteBuffer readBloomFilter(StorageConfiguration<?> conf, StoragePath filePath) throws IOException {
- HoodieConfig hoodieConfig = getReaderConfigs(conf);
- try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(conf).getReaderFactory(HoodieRecordType.AVRO)
+ private static ByteBuffer readBloomFilter(HoodieStorage storage, StoragePath filePath) throws IOException {
+ HoodieConfig hoodieConfig = getReaderConfigs(storage.getConf());
+ try (HoodieFileReader fileReader = HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, filePath)) {
final BloomFilter fileBloomFilter = fileReader.readBloomFilter();
if (fileBloomFilter == null) {
@@ -1176,7 +1176,7 @@ public class HoodieTableMetadataUtil {
if (filePath.endsWith(HoodieFileFormat.PARQUET.getFileExtension())) {
StoragePath fullFilePath = new StoragePath(datasetMetaClient.getBasePathV2(), filePath);
return FileFormatUtils.getInstance(HoodieFileFormat.PARQUET)
- .readColumnStatsFromMetadata(datasetMetaClient.getStorageConf(), fullFilePath, columnsToIndex);
+ .readColumnStatsFromMetadata(datasetMetaClient.getStorage(), fullFilePath, columnsToIndex);
}
LOG.warn("Column range index not supported for: {}", filePath);
@@ -1449,8 +1449,7 @@ public class HoodieTableMetadataUtil {
public static String deleteMetadataTable(HoodieTableMetaClient dataMetaClient, HoodieEngineContext context, boolean backup) {
final StoragePath metadataTablePath =
HoodieTableMetadata.getMetadataTableBasePath(dataMetaClient.getBasePathV2());
- HoodieStorage storage = HoodieStorageUtils.getStorage(
- metadataTablePath.toString(), context.getStorageConf());
+ HoodieStorage storage = dataMetaClient.getStorage();
dataMetaClient.getTableConfig().clearMetadataPartitions(dataMetaClient);
try {
if (!storage.exists(metadataTablePath)) {
@@ -1505,7 +1504,7 @@ public class HoodieTableMetadataUtil {
}
final StoragePath metadataTablePartitionPath = new StoragePath(HoodieTableMetadata.getMetadataTableBasePath(dataMetaClient.getBasePath()), partitionType.getPartitionPath());
- HoodieStorage storage = HoodieStorageUtils.getStorage(metadataTablePartitionPath.toString(), context.getStorageConf());
+ HoodieStorage storage = dataMetaClient.getStorage();
dataMetaClient.getTableConfig().setMetadataPartitionState(dataMetaClient, partitionType, false);
try {
if (!storage.exists(metadataTablePartitionPath)) {
@@ -1780,7 +1779,8 @@ public class HoodieTableMetadataUtil {
final String fileId = baseFile.getFileId();
final String instantTime = baseFile.getCommitTime();
- HoodieFileReader reader = HoodieIOFactory.getIOFactory(configuration).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
+ HoodieFileReader reader = HoodieIOFactory.getIOFactory(HoodieStorageUtils.getStorage(basePath, configuration))
+ .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(config, dataFilePath);
return getHoodieRecordIterator(reader.getRecordKeyIterator(), forDelete, partition, fileId, instantTime);
});
@@ -1841,7 +1841,8 @@ public class HoodieTableMetadataUtil {
final String fileId = baseFile.getFileId();
final String instantTime = baseFile.getCommitTime();
HoodieConfig hoodieConfig = getReaderConfigs(storageConf);
- HoodieFileReader reader = HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
+ HoodieFileReader reader = HoodieIOFactory.getIOFactory(metaClient.getStorage())
+ .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, dataFilePath);
return getHoodieRecordIterator(reader.getRecordKeyIterator(), forDelete, partition, fileId, instantTime);
});
diff --git a/hudi-common/src/main/java/org/apache/hudi/metrics/Metrics.java b/hudi-common/src/main/java/org/apache/hudi/metrics/Metrics.java
index cc50d3a4147..441c0b5b002 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metrics/Metrics.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metrics/Metrics.java
@@ -24,8 +24,6 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import com.codahale.metrics.MetricRegistry;
@@ -53,10 +51,10 @@ public class Metrics {
private final String basePath;
private boolean initialized = false;
private transient Thread shutdownThread = null;
- private final StorageConfiguration<?> storageConf;
+ private final HoodieStorage storage;
- public Metrics(HoodieMetricsConfig metricConfig, StorageConfiguration<?> storageConf) {
- this.storageConf = storageConf;
+ public Metrics(HoodieMetricsConfig metricConfig, HoodieStorage storage) {
+ this.storage = storage;
registry = new MetricRegistry();
commonMetricPrefix = metricConfig.getMetricReporterMetricsNamePrefix();
reporters = new ArrayList<>();
@@ -80,13 +78,13 @@ public class Metrics {
registerGauges(Registry.getAllMetrics(true, true), Option.of(commonMetricPrefix));
}
- public static synchronized Metrics getInstance(HoodieMetricsConfig metricConfig, StorageConfiguration<?> storageConf) {
+ public static synchronized Metrics getInstance(HoodieMetricsConfig metricConfig, HoodieStorage storage) {
String basePath = getBasePath(metricConfig);
if (METRICS_INSTANCE_PER_BASEPATH.containsKey(basePath)) {
return METRICS_INSTANCE_PER_BASEPATH.get(basePath);
}
- Metrics metrics = new Metrics(metricConfig, storageConf);
+ Metrics metrics = new Metrics(metricConfig, storage);
METRICS_INSTANCE_PER_BASEPATH.put(basePath, metrics);
return metrics;
}
@@ -100,10 +98,10 @@ public class Metrics {
private List<MetricsReporter> addAdditionalMetricsExporters(HoodieMetricsConfig metricConfig) {
List<MetricsReporter> reporterList = new ArrayList<>();
List<String> propPathList = StringUtils.split(metricConfig.getMetricReporterFileBasedConfigs(), ",");
- try (HoodieStorage storage = HoodieStorageUtils.getStorage(propPathList.get(0), storageConf)) {
+ try (HoodieStorage newStorage = storage.newInstance(new StoragePath(propPathList.get(0)), storage.getConf())) {
for (String propPath : propPathList) {
HoodieMetricsConfig secondarySourceConfig = HoodieMetricsConfig.newBuilder().fromInputStream(
- storage.open(new StoragePath(propPath))).withPath(metricConfig.getBasePath()).build();
+ newStorage.open(new StoragePath(propPath))).withPath(metricConfig.getBasePath()).build();
Option<MetricsReporter> reporter = MetricsReporterFactory.createReporter(secondarySourceConfig, registry);
if (reporter.isPresent()) {
reporterList.add(reporter.get());
diff --git a/hudi-common/src/main/java/org/apache/hudi/storage/HoodieStorageUtils.java b/hudi-common/src/main/java/org/apache/hudi/storage/HoodieStorageUtils.java
index 770fc77372e..6946034adf4 100644
--- a/hudi-common/src/main/java/org/apache/hudi/storage/HoodieStorageUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/storage/HoodieStorageUtils.java
@@ -19,7 +19,9 @@
package org.apache.hudi.storage;
-import static org.apache.hudi.io.storage.HoodieIOFactory.getIOFactory;
+import org.apache.hudi.common.config.HoodieStorageConfig;
+import org.apache.hudi.common.util.ReflectionUtils;
+import org.apache.hudi.exception.HoodieException;
public class HoodieStorageUtils {
public static final String DEFAULT_URI = "file:///";
@@ -33,6 +35,13 @@ public class HoodieStorageUtils {
}
public static HoodieStorage getStorage(StoragePath path, StorageConfiguration<?> conf) {
- return getIOFactory(conf).getStorage(path);
+ String storageClass = conf.getString(HoodieStorageConfig.HOODIE_STORAGE_CLASS.key())
+ .orElse(HoodieStorageConfig.HOODIE_STORAGE_CLASS.defaultValue());
+ try {
+ return (HoodieStorage) ReflectionUtils.loadClass(
+ storageClass, new Class<?>[] {StoragePath.class, StorageConfiguration.class}, path, conf);
+ } catch (Exception e) {
+ throw new HoodieException("Unable to create " + storageClass, e);
+ }
}
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
index 074d9b1c020..62a0c861a9f 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
@@ -51,6 +51,8 @@ import java.util.Objects;
import java.util.Properties;
import java.util.UUID;
+import static org.apache.hudi.storage.HoodieStorageUtils.DEFAULT_URI;
+
/**
* A utility class for testing.
*/
@@ -73,6 +75,10 @@ public class HoodieTestUtils {
new Class<?>[] {Boolean.class}, true);
}
+ public static HoodieStorage getDefaultStorage() {
+ return getStorage(DEFAULT_URI);
+ }
+
public static HoodieStorage getStorage(String path) {
return HoodieStorageUtils.getStorage(path, getDefaultStorageConf());
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/metadata/HoodieBackedTestDelayedTableMetadata.java b/hudi-common/src/test/java/org/apache/hudi/metadata/HoodieBackedTestDelayedTableMetadata.java
index b195083a201..51a300ef582 100644
--- a/hudi-common/src/test/java/org/apache/hudi/metadata/HoodieBackedTestDelayedTableMetadata.java
+++ b/hudi-common/src/test/java/org/apache/hudi/metadata/HoodieBackedTestDelayedTableMetadata.java
@@ -21,6 +21,7 @@ package org.apache.hudi.metadata;
import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.storage.HoodieStorage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,10 +34,11 @@ public class HoodieBackedTestDelayedTableMetadata extends HoodieBackedTableMetad
private static final Logger LOG = LoggerFactory.getLogger(HoodieBackedTestDelayedTableMetadata.class);
public HoodieBackedTestDelayedTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
HoodieMetadataConfig metadataConfig,
String datasetBasePath,
boolean reuse) {
- super(engineContext, metadataConfig, datasetBasePath, reuse);
+ super(engineContext, storage, metadataConfig, datasetBasePath, reuse);
}
@Override
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/BootstrapOperator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/BootstrapOperator.java
index d98470e6444..e654209e87b 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/BootstrapOperator.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/BootstrapOperator.java
@@ -38,7 +38,6 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.configuration.HadoopConfigurations;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.sink.bootstrap.aggregate.BootstrapAggFunction;
import org.apache.hudi.sink.meta.CkpMetadata;
import org.apache.hudi.table.HoodieTable;
@@ -144,7 +143,8 @@ public class BootstrapOperator<I, O extends HoodieRecord<?>>
String basePath = hoodieTable.getMetaClient().getBasePath();
int taskID = getRuntimeContext().getIndexOfThisSubtask();
LOG.info("Start loading records in table {} into the index state, taskId = {}", basePath, taskID);
- for (String partitionPath : FSUtils.getAllPartitionPaths(new HoodieFlinkEngineContext(hadoopConf), metadataConfig(conf), basePath)) {
+ for (String partitionPath : FSUtils.getAllPartitionPaths(
+ new HoodieFlinkEngineContext(hadoopConf), hoodieTable.getStorage(), metadataConfig(conf), basePath)) {
if (pattern.matcher(partitionPath).matches()) {
loadRecords(partitionPath);
}
@@ -220,7 +220,7 @@ public class BootstrapOperator<I, O extends HoodieRecord<?>>
return;
}
try (ClosableIterator<HoodieKey> iterator = fileUtils.getHoodieKeyIterator(
- HadoopFSUtils.getStorageConf(this.hadoopConf), baseFile.getStoragePath())) {
+ hoodieTable.getStorage(), baseFile.getStoragePath())) {
iterator.forEachRemaining(hoodieKey -> {
output.collect(new StreamRecord(new IndexRecord(generateHoodieRecord(hoodieKey, fileSlice))));
});
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java
index 460e36154cf..2e1bfd604d7 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/clustering/ClusteringOperator.java
@@ -274,11 +274,11 @@ public class ClusteringOperator extends TableStreamOperator<ClusteringCommitEven
try {
Option<HoodieFileReader> baseFileReader = StringUtils.isNullOrEmpty(clusteringOp.getDataFilePath())
? Option.empty()
- : Option.of(HoodieIOFactory.getIOFactory(table.getStorageConf())
+ : Option.of(HoodieIOFactory.getIOFactory(table.getStorage())
.getReaderFactory(table.getConfig().getRecordMerger().getRecordType())
.getFileReader(table.getConfig(), new StoragePath(clusteringOp.getDataFilePath())));
HoodieMergedLogRecordScanner scanner = HoodieMergedLogRecordScanner.newBuilder()
- .withStorage(table.getMetaClient().getStorage())
+ .withStorage(table.getStorage())
.withBasePath(table.getMetaClient().getBasePath())
.withLogFilePaths(clusteringOp.getDeltaFilePaths())
.withReaderSchema(readerSchema)
@@ -322,7 +322,7 @@ public class ClusteringOperator extends TableStreamOperator<ClusteringCommitEven
List<Iterator<RowData>> iteratorsForPartition = clusteringOps.stream().map(clusteringOp -> {
Iterable<IndexedRecord> indexedRecords = () -> {
try {
- HoodieFileReaderFactory fileReaderFactory = HoodieIOFactory.getIOFactory(table.getStorageConf())
+ HoodieFileReaderFactory fileReaderFactory = HoodieIOFactory.getIOFactory(table.getStorage())
.getReaderFactory(table.getConfig().getRecordMerger().getRecordType());
HoodieAvroFileReader fileReader = (HoodieAvroFileReader) fileReaderFactory.getFileReader(
table.getConfig(), new StoragePath(clusteringOp.getDataFilePath()));
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfiles.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfiles.java
index 1536dae35ba..b2c7d56f624 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfiles.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfiles.java
@@ -31,6 +31,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.StreamerUtil;
import org.apache.flink.core.fs.Path;
@@ -148,9 +149,9 @@ public class WriteProfiles {
) {
switch (tableType) {
case COPY_ON_WRITE:
- return metadata.getFileIdToInfo(HadoopFSUtils.getStorageConf(hadoopConf), basePath);
+ return metadata.getFileIdToInfo(basePath);
case MERGE_ON_READ:
- return metadata.getFullPathToInfo(HadoopFSUtils.getStorageConf(hadoopConf), basePath);
+ return metadata.getFullPathToInfo(new HoodieHadoopStorage(basePath, hadoopConf), basePath);
default:
throw new AssertionError();
}
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java
index a1461975811..649a552a7b4 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java
@@ -29,8 +29,9 @@ import org.apache.hudi.source.prune.DataPruner;
import org.apache.hudi.source.prune.PartitionPruners;
import org.apache.hudi.source.prune.PrimaryKeyPruners;
import org.apache.hudi.source.stats.ColumnStatsIndices;
-import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.DataTypeUtils;
import org.apache.hudi.util.StreamerUtil;
@@ -150,7 +151,8 @@ public class FileIndex {
String[] partitions =
getOrBuildPartitionPaths().stream().map(p -> fullPartitionPath(path, p)).toArray(String[]::new);
List<StoragePathInfo> allFiles = FSUtils.getFilesInPartitions(
- new HoodieFlinkEngineContext(hadoopConf), metadataConfig, path.toString(), partitions)
+ new HoodieFlinkEngineContext(hadoopConf),
+ new HoodieHadoopStorage(path, hadoopConf), metadataConfig, path.toString(), partitions)
.values().stream()
.flatMap(e -> e.stream())
.collect(Collectors.toList());
@@ -278,7 +280,8 @@ public class FileIndex {
return this.partitionPaths;
}
List<String> allPartitionPaths = this.tableExists
- ? FSUtils.getAllPartitionPaths(new HoodieFlinkEngineContext(hadoopConf), metadataConfig, path.toString())
+ ? FSUtils.getAllPartitionPaths(new HoodieFlinkEngineContext(hadoopConf),
+ new HoodieHadoopStorage(path, hadoopConf), metadataConfig, path.toString())
: Collections.emptyList();
if (this.partitionPruner == null) {
this.partitionPaths = allPartitionPaths;
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java
index 7032f299368..674e58dce6b 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java
@@ -31,8 +31,10 @@ import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.metadata.HoodieMetadataPayload;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.AvroSchemaConverter;
import org.apache.hudi.util.AvroToRowDataConverters;
+import org.apache.hudi.util.FlinkClientUtil;
import org.apache.hudi.util.RowDataProjection;
import org.apache.avro.generic.GenericRecord;
@@ -285,7 +287,7 @@ public class ColumnStatsIndices {
// - Fetching the records from CSI by key-prefixes (encoded column names)
// - Deserializing fetched records into [[RowData]]s
HoodieTableMetadata metadataTable = HoodieTableMetadata.create(
- HoodieFlinkEngineContext.DEFAULT,
+ HoodieFlinkEngineContext.DEFAULT, new HoodieHadoopStorage(basePath, FlinkClientUtil.getHadoopConf()),
metadataConfig, basePath);
// TODO encoding should be done internally w/in HoodieBackedTableMetadata
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/InternalSchemaManager.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/InternalSchemaManager.java
index 9203e6dd11b..c5726b9d844 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/InternalSchemaManager.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/InternalSchemaManager.java
@@ -27,13 +27,13 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.configuration.HadoopConfigurations;
import org.apache.hudi.configuration.OptionsResolver;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.internal.schema.InternalSchema;
import org.apache.hudi.internal.schema.Type;
import org.apache.hudi.internal.schema.Types;
import org.apache.hudi.internal.schema.action.InternalSchemaMerger;
import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter;
import org.apache.hudi.internal.schema.utils.InternalSchemaUtils;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.AvroSchemaConverter;
import org.apache.flink.configuration.Configuration;
@@ -111,7 +111,9 @@ public class InternalSchemaManager implements Serializable {
}
long commitInstantTime = Long.parseLong(FSUtils.getCommitTime(fileName));
InternalSchema fileSchema = InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, HadoopFSUtils.getStorageConf(getHadoopConf()), validCommits);
+ commitInstantTime, tablePath,
+ new HoodieHadoopStorage(tablePath, getHadoopConf()),
+ validCommits);
if (querySchema.equals(fileSchema)) {
return InternalSchema.getEmptyInternalSchema();
}
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java
index 57966b4bdbf..8cd8be8b5cc 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcInputFormat.java
@@ -35,12 +35,11 @@ import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.configuration.OptionsResolver;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.internal.schema.InternalSchema;
import org.apache.hudi.source.ExpressionPredicates.Predicate;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.table.format.FormatUtils;
import org.apache.hudi.table.format.InternalSchemaManager;
import org.apache.hudi.table.format.mor.MergeOnReadInputFormat;
@@ -336,7 +335,7 @@ public class CdcInputFormat extends MergeOnReadInputFormat {
this.recordBuilder = new GenericRecordBuilder(requiredSchema);
this.avroToRowDataConverter = AvroToRowDataConverters.createRowConverter(tableState.getRequiredRowType());
StoragePath hadoopTablePath = new StoragePath(tablePath);
- HoodieStorage storage = HoodieStorageUtils.getStorage(hadoopTablePath, HadoopFSUtils.getStorageConf(hadoopConf));
+ HoodieStorage storage = new HoodieHadoopStorage(tablePath, hadoopConf);
HoodieLogFile[] cdcLogFiles = fileSplit.getCdcFiles().stream().map(cdcFile -> {
try {
return new HoodieLogFile(
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
index 6ecf1b33045..20a3a9d9ccf 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
@@ -243,7 +243,7 @@ public class TestStreamWriteOperatorCoordinator {
String basePath = tempFile.getAbsolutePath();
HoodieStorage storage =
- coordinator.getWriteClient().getHoodieTable().getMetaClient().getStorage();
+ coordinator.getWriteClient().getHoodieTable().getStorage();
assertTrue(HoodieHeartbeatClient.heartbeatExists(storage, basePath, instant),
"Heartbeat is existed");
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java
index b925a895628..5d0de6c442f 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java
@@ -423,7 +423,7 @@ public class ITTestHoodieFlinkCompactor {
Set<Pair<String, String>> fileIdCommitTimeSet = new HashSet<>();
HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(conf);
HoodieStorage storage = metaClient.getStorage();
- FSUtils.getAllPartitionPaths(HoodieFlinkEngineContext.DEFAULT, metaClient.getBasePath(), false, false).forEach(
+ FSUtils.getAllPartitionPaths(HoodieFlinkEngineContext.DEFAULT, metaClient.getStorage(), metaClient.getBasePath(), false, false).forEach(
partition -> {
try {
storage.listDirectEntries(FSUtils.constructAbsolutePath(metaClient.getBasePathV2(), partition))
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
index 2e3f546debe..3941c28e570 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hadoop.conf.Configuration;
import org.slf4j.Logger;
@@ -147,7 +148,7 @@ public class DFSPropertiesConfiguration extends PropertiesConfig {
throw new IllegalStateException("Loop detected; file " + filePath + " already referenced");
}
- HoodieStorage storage = HoodieStorageUtils.getStorage(
+ HoodieStorage storage = new HoodieHadoopStorage(
filePath,
HadoopFSUtils.getStorageConf(Option.ofNullable(hadoopConfig).orElseGet(Configuration::new))
);
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
index ca7b30d7d03..8e8c19b3c7b 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
@@ -160,7 +160,7 @@ public class HoodieLogFormatWriter implements HoodieLogFormat.Writer {
// bytes for header
byte[] headerBytes = HoodieLogBlock.getLogMetadataBytes(block.getLogBlockHeader());
// content bytes
- byte[] content = block.getContentBytes(storage.getConf());
+ byte[] content = block.getContentBytes(storage);
// bytes for footer
byte[] footerBytes = HoodieLogBlock.getLogMetadataBytes(block.getLogBlockFooter());
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
index aa691be3573..0ab01b18a79 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
@@ -33,7 +33,6 @@ import org.apache.hudi.io.storage.HoodieHBaseKVComparator;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -84,56 +83,56 @@ public class HFileUtils extends FileFormatUtils {
}
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath) {
throw new UnsupportedOperationException("HFileUtils does not support readAvroRecords");
}
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath, Schema schema) {
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath, Schema schema) {
throw new UnsupportedOperationException("HFileUtils does not support readAvroRecords");
}
@Override
- public Map<String, String> readFooter(StorageConfiguration<?> configuration, boolean required, StoragePath filePath, String... footerNames) {
+ public Map<String, String> readFooter(HoodieStorage storage, boolean required, StoragePath filePath, String... footerNames) {
throw new UnsupportedOperationException("HFileUtils does not support readFooter");
}
@Override
- public long getRowCount(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public long getRowCount(HoodieStorage storage, StoragePath filePath) {
throw new UnsupportedOperationException("HFileUtils does not support getRowCount");
}
@Override
- public Set<String> filterRowKeys(StorageConfiguration<?> configuration, StoragePath filePath, Set<String> filter) {
+ public Set<String> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set<String> filter) {
throw new UnsupportedOperationException("HFileUtils does not support filterRowKeys");
}
@Override
- public List<HoodieKey> fetchHoodieKeys(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public List<HoodieKey> fetchHoodieKeys(HoodieStorage storage, StoragePath filePath) {
throw new UnsupportedOperationException("HFileUtils does not support fetchRecordKeysWithPositions");
}
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
throw new UnsupportedOperationException("HFileUtils does not support getHoodieKeyIterator");
}
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath) {
throw new UnsupportedOperationException("HFileUtils does not support getHoodieKeyIterator");
}
@Override
- public List<HoodieKey> fetchHoodieKeys(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public List<HoodieKey> fetchHoodieKeys(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
throw new UnsupportedOperationException("HFileUtils does not support fetchRecordKeysWithPositions");
}
@Override
- public Schema readAvroSchema(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public Schema readAvroSchema(HoodieStorage storage, StoragePath filePath) {
LOG.info("Reading schema from {}", filePath);
try (HoodieFileReader fileReader =
- HoodieIOFactory.getIOFactory(configuration)
+ HoodieIOFactory.getIOFactory(storage)
.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(
ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER,
@@ -145,7 +144,7 @@ public class HFileUtils extends FileFormatUtils {
}
@Override
- public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(StorageConfiguration<?> storageConf, StoragePath filePath, List<String> columnList) {
+ public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(HoodieStorage storage, StoragePath filePath, List<String> columnList) {
throw new UnsupportedOperationException(
"Reading column statistics from metadata is not supported for HFile format yet");
}
@@ -161,7 +160,7 @@ public class HFileUtils extends FileFormatUtils {
}
@Override
- public byte[] serializeRecordsToLogBlock(StorageConfiguration<?> storageConf,
+ public byte[] serializeRecordsToLogBlock(HoodieStorage storage,
List<HoodieRecord> records,
Schema writerSchema,
Schema readerSchema,
@@ -174,7 +173,7 @@ public class HFileUtils extends FileFormatUtils {
.withCellComparator(new HoodieHBaseKVComparator())
.build();
- Configuration conf = storageConf.unwrapAs(Configuration.class);
+ Configuration conf = storage.getConf().unwrapAs(Configuration.class);
CacheConfig cacheConfig = new CacheConfig(conf);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
FSDataOutputStream ostream = new FSDataOutputStream(baos, null);
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
index 03d72f4a97b..8233f954c1f 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
@@ -31,8 +31,6 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.hadoop.OrcReaderIterator;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -74,14 +72,14 @@ public class OrcUtils extends FileFormatUtils {
/**
* Provides a closable iterator for reading the given ORC file.
*
- * @param configuration configuration to build storage object
- * @param filePath The ORC file path
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath The ORC file path
* @return {@link ClosableIterator} of {@link HoodieKey}s for reading the ORC file
*/
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath) {
try {
- Configuration conf = configuration.unwrapCopyAs(Configuration.class);
+ Configuration conf = storage.getConf().unwrapCopyAs(Configuration.class);
conf.addResource(HadoopFSUtils.getFs(filePath.toString(), conf).getConf());
Reader reader = OrcFile.createReader(convertToHadoopPath(filePath), OrcFile.readerOptions(conf));
@@ -113,33 +111,33 @@ public class OrcUtils extends FileFormatUtils {
/**
* Fetch {@link HoodieKey}s from the given ORC file.
*
- * @param configuration configuration to build storage object
- * @param filePath The ORC file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath The ORC file path.
* @return {@link List} of {@link HoodieKey}s fetched from the ORC file
*/
@Override
- public List<HoodieKey> fetchHoodieKeys(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public List<HoodieKey> fetchHoodieKeys(HoodieStorage storage, StoragePath filePath) {
try {
- if (!HoodieStorageUtils.getStorage(filePath, configuration).exists(filePath)) {
+ if (!storage.exists(filePath)) {
return Collections.emptyList();
}
} catch (IOException e) {
throw new HoodieIOException("Failed to read from ORC file:" + filePath, e);
}
List<HoodieKey> hoodieKeys = new ArrayList<>();
- try (ClosableIterator<HoodieKey> iterator = getHoodieKeyIterator(configuration, filePath, Option.empty())) {
+ try (ClosableIterator<HoodieKey> iterator = getHoodieKeyIterator(storage, filePath, Option.empty())) {
iterator.forEachRemaining(hoodieKeys::add);
}
return hoodieKeys;
}
@Override
- public List<HoodieKey> fetchHoodieKeys(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public List<HoodieKey> fetchHoodieKeys(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
throw new UnsupportedOperationException("Custom key generator is not supported yet");
}
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
throw new UnsupportedOperationException("Custom key generator is not supported yet");
}
@@ -147,28 +145,28 @@ public class OrcUtils extends FileFormatUtils {
* NOTE: This literally reads the entire file contents, thus should be used with caution.
*/
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath) {
Schema avroSchema;
try (Reader reader = OrcFile.createReader(
- convertToHadoopPath(filePath), OrcFile.readerOptions(configuration.unwrapAs(Configuration.class)))) {
+ convertToHadoopPath(filePath), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) {
avroSchema = AvroOrcUtils.createAvroSchema(reader.getSchema());
} catch (IOException io) {
throw new HoodieIOException("Unable to read Avro records from an ORC file:" + filePath, io);
}
- return readAvroRecords(configuration, filePath, avroSchema);
+ return readAvroRecords(storage, filePath, avroSchema);
}
/**
* NOTE: This literally reads the entire file contents, thus should be used with caution.
*/
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath, Schema avroSchema) {
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath, Schema avroSchema) {
List<GenericRecord> records = new ArrayList<>();
try (Reader reader = OrcFile.createReader(
- convertToHadoopPath(filePath), OrcFile.readerOptions(configuration.unwrapAs(Configuration.class)))) {
+ convertToHadoopPath(filePath), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) {
TypeDescription orcSchema = reader.getSchema();
try (RecordReader recordReader = reader.rows(
- new Options(configuration.unwrapAs(Configuration.class)).schema(orcSchema))) {
+ new Options(storage.getConf().unwrapAs(Configuration.class)).schema(orcSchema))) {
OrcReaderIterator<GenericRecord> iterator = new OrcReaderIterator<>(recordReader, avroSchema, orcSchema);
while (iterator.hasNext()) {
GenericRecord record = iterator.next();
@@ -185,17 +183,17 @@ public class OrcUtils extends FileFormatUtils {
* Read the rowKey list matching the given filter, from the given ORC file. If the filter is empty, then this will
* return all the rowkeys.
*
- * @param conf configuration to build storage object.
+ * @param storage {@link HoodieStorage} instance.
* @param filePath The ORC file path.
* @param filter record keys filter
* @return Set of row keys matching candidateRecordKeys
*/
@Override
- public Set<String> filterRowKeys(StorageConfiguration<?> conf, StoragePath filePath, Set<String> filter)
+ public Set<String> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set<String> filter)
throws HoodieIOException {
- try (Reader reader = OrcFile.createReader(new Path(filePath.toUri()), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)));) {
+ try (Reader reader = OrcFile.createReader(new Path(filePath.toUri()), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)));) {
TypeDescription schema = reader.getSchema();
- try (RecordReader recordReader = reader.rows(new Options(conf.unwrapAs(Configuration.class)).schema(schema))) {
+ try (RecordReader recordReader = reader.rows(new Options(storage.getConf().unwrapAs(Configuration.class)).schema(schema))) {
Set<String> filteredRowKeys = new HashSet<>();
List<String> fieldNames = schema.getFieldNames();
VectorizedRowBatch batch = schema.createRowBatch();
@@ -228,10 +226,10 @@ public class OrcUtils extends FileFormatUtils {
}
@Override
- public Map<String, String> readFooter(StorageConfiguration<?> conf, boolean required,
+ public Map<String, String> readFooter(HoodieStorage storage, boolean required,
StoragePath filePath, String... footerNames) {
try (Reader reader = OrcFile.createReader(
- convertToHadoopPath(filePath), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)))) {
+ convertToHadoopPath(filePath), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) {
Map<String, String> footerVals = new HashMap<>();
List<UserMetadataItem> metadataItemList = reader.getFileTail().getFooter().getMetadataList();
Map<String, String> metadata = metadataItemList.stream().collect(Collectors.toMap(
@@ -252,9 +250,9 @@ public class OrcUtils extends FileFormatUtils {
}
@Override
- public Schema readAvroSchema(StorageConfiguration<?> conf, StoragePath filePath) {
+ public Schema readAvroSchema(HoodieStorage storage, StoragePath filePath) {
try (Reader reader = OrcFile.createReader(
- convertToHadoopPath(filePath), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)))) {
+ convertToHadoopPath(filePath), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) {
if (reader.hasMetadataValue("orc.avro.schema")) {
ByteBuffer metadataValue = reader.getMetadataValue("orc.avro.schema");
byte[] bytes = toBytes(metadataValue);
@@ -269,7 +267,7 @@ public class OrcUtils extends FileFormatUtils {
}
@Override
- public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(StorageConfiguration<?> storageConf, StoragePath filePath, List<String> columnList) {
+ public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(HoodieStorage storage, StoragePath filePath, List<String> columnList) {
throw new UnsupportedOperationException(
"Reading column statistics from metadata is not supported for ORC format yet");
}
@@ -280,9 +278,9 @@ public class OrcUtils extends FileFormatUtils {
}
@Override
- public long getRowCount(StorageConfiguration<?> conf, StoragePath filePath) {
+ public long getRowCount(HoodieStorage storage, StoragePath filePath) {
try (Reader reader = OrcFile.createReader(
- convertToHadoopPath(filePath), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)))) {
+ convertToHadoopPath(filePath), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) {
return reader.getNumberOfRows();
} catch (IOException io) {
throw new HoodieIOException("Unable to get row count for ORC file:" + filePath, io);
@@ -305,7 +303,7 @@ public class OrcUtils extends FileFormatUtils {
}
@Override
- public byte[] serializeRecordsToLogBlock(StorageConfiguration<?> storageConf,
+ public byte[] serializeRecordsToLogBlock(HoodieStorage storage,
List<HoodieRecord> records,
Schema writerSchema,
Schema readerSchema, String keyFieldName,
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
index ad42567e647..1dad2d237cf 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
@@ -33,8 +33,6 @@ import org.apache.hudi.io.storage.HoodieFileWriter;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -81,6 +79,7 @@ import java.util.stream.Stream;
import static org.apache.hudi.common.config.HoodieStorageConfig.PARQUET_BLOCK_SIZE;
import static org.apache.hudi.common.config.HoodieStorageConfig.PARQUET_MAX_FILE_SIZE;
import static org.apache.hudi.common.config.HoodieStorageConfig.PARQUET_PAGE_SIZE;
+import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath;
/**
* Utility functions involving with parquet.
@@ -93,23 +92,23 @@ public class ParquetUtils extends FileFormatUtils {
* Read the rowKey list matching the given filter, from the given parquet file. If the filter is empty, then this will
* return all the rowkeys.
*
- * @param configuration configuration to build storage object
- * @param filePath The parquet file path.
- * @param filter record keys filter
- * @return Set Set of row keys matching candidateRecordKeys
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath The parquet file path.
+ * @param filter record keys filter
+ * @return Set of row keys matching candidateRecordKeys
*/
@Override
- public Set<String> filterRowKeys(StorageConfiguration<?> configuration, StoragePath filePath, Set<String> filter) {
- return filterParquetRowKeys(configuration, new Path(filePath.toUri()), filter, HoodieAvroUtils.getRecordKeySchema());
+ public Set<String> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set<String> filter) {
+ return filterParquetRowKeys(storage, new Path(filePath.toUri()), filter, HoodieAvroUtils.getRecordKeySchema());
}
- public static ParquetMetadata readMetadata(StorageConfiguration<?> conf, StoragePath parquetFilePath) {
+ public static ParquetMetadata readMetadata(HoodieStorage storage, StoragePath parquetFilePath) {
Path parquetFileHadoopPath = new Path(parquetFilePath.toUri());
ParquetMetadata footer;
try {
// TODO(vc): Should we use the parallel reading version here?
- footer = ParquetFileReader.readFooter(HoodieStorageUtils.getStorage(
- parquetFileHadoopPath.toString(), conf).getConf().unwrapAs(Configuration.class), parquetFileHadoopPath);
+ footer = ParquetFileReader.readFooter(storage.newInstance(
+ parquetFilePath, storage.getConf()).getConf().unwrapAs(Configuration.class), parquetFileHadoopPath);
} catch (IOException e) {
throw new HoodieIOException("Failed to read footer for parquet " + parquetFileHadoopPath, e);
}
@@ -120,20 +119,20 @@ public class ParquetUtils extends FileFormatUtils {
* Read the rowKey list matching the given filter, from the given parquet file. If the filter is empty, then this will
* return all the rowkeys.
*
- * @param filePath The parquet file path.
- * @param configuration configuration to build storage object
- * @param filter record keys filter
- * @param readSchema schema of columns to be read
- * @return Set Set of row keys matching candidateRecordKeys
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath The parquet file path.
+ * @param filter record keys filter
+ * @param readSchema schema of columns to be read
+ * @return Set of row keys matching candidateRecordKeys
*/
- private static Set<String> filterParquetRowKeys(StorageConfiguration<?> configuration, Path filePath, Set<String> filter,
+ private static Set<String> filterParquetRowKeys(HoodieStorage storage, Path filePath, Set<String> filter,
Schema readSchema) {
Option<RecordKeysFilterFunction> filterFunction = Option.empty();
if (filter != null && !filter.isEmpty()) {
filterFunction = Option.of(new RecordKeysFilterFunction(filter));
}
- Configuration conf = configuration.unwrapCopyAs(Configuration.class);
- conf.addResource(HoodieStorageUtils.getStorage(filePath.toString(), configuration).getConf().unwrapAs(Configuration.class));
+ Configuration conf = storage.getConf().unwrapCopyAs(Configuration.class);
+ conf.addResource(storage.newInstance(convertToStoragePath(filePath), storage.getConf()).getConf().unwrapAs(Configuration.class));
AvroReadSupport.setAvroReadSchema(conf, readSchema);
AvroReadSupport.setRequestedProjection(conf, readSchema);
Set<String> rowKeys = new HashSet<>();
@@ -167,39 +166,39 @@ public class ParquetUtils extends FileFormatUtils {
/**
* Fetch {@link HoodieKey}s from the given parquet file.
*
- * @param configuration configuration to build storage object
- * @param filePath The parquet file path.
+ * @param storage {@link HoodieStorage} instance.
+ * @param filePath The parquet file path.
* @return {@link List} of {@link HoodieKey}s fetched from the parquet file
*/
@Override
- public List<HoodieKey> fetchHoodieKeys(StorageConfiguration<?> configuration, StoragePath filePath) {
- return fetchHoodieKeys(configuration, filePath, Option.empty());
+ public List<HoodieKey> fetchHoodieKeys(HoodieStorage storage, StoragePath filePath) {
+ return fetchHoodieKeys(storage, filePath, Option.empty());
}
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath) {
- return getHoodieKeyIterator(configuration, filePath, Option.empty());
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath) {
+ return getHoodieKeyIterator(storage, filePath, Option.empty());
}
/**
* Returns a closable iterator for reading the given parquet file.
*
- * @param configuration configuration to build storage object
+ * @param storage {@link HoodieStorage} instance.
* @param filePath The parquet file path
* @param keyGeneratorOpt instance of KeyGenerator
* @return {@link ClosableIterator} of {@link HoodieKey}s for reading the parquet file
*/
@Override
- public ClosableIterator<HoodieKey> getHoodieKeyIterator(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
try {
- Configuration conf = configuration.unwrapCopyAs(Configuration.class);
- conf.addResource(HoodieStorageUtils.getStorage(filePath.toString(), configuration).getConf().unwrapAs(Configuration.class));
+ Configuration conf = storage.getConf().unwrapCopyAs(Configuration.class);
+ conf.addResource(storage.newInstance(filePath, storage.getConf()).getConf().unwrapAs(Configuration.class));
Schema readSchema = keyGeneratorOpt
.map(keyGenerator -> {
List<String> fields = new ArrayList<>();
fields.addAll(keyGenerator.getRecordKeyFieldNames());
fields.addAll(keyGenerator.getPartitionPathFields());
- return HoodieAvroUtils.getSchemaForFields(readAvroSchema(configuration, filePath), fields);
+ return HoodieAvroUtils.getSchemaForFields(readAvroSchema(storage, filePath), fields);
})
.orElse(HoodieAvroUtils.getRecordKeyPartitionPathSchema());
AvroReadSupport.setAvroReadSchema(conf, readSchema);
@@ -215,15 +214,15 @@ public class ParquetUtils extends FileFormatUtils {
/**
* Fetch {@link HoodieKey}s from the given parquet file.
*
- * @param configuration configuration to build storage object
+ * @param storage {@link HoodieStorage} instance.
* @param filePath The parquet file path.
* @param keyGeneratorOpt instance of KeyGenerator.
* @return {@link List} of {@link HoodieKey}s fetched from the parquet file
*/
@Override
- public List<HoodieKey> fetchHoodieKeys(StorageConfiguration<?> configuration, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
+ public List<HoodieKey> fetchHoodieKeys(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt) {
List<HoodieKey> hoodieKeys = new ArrayList<>();
- try (ClosableIterator<HoodieKey> iterator = getHoodieKeyIterator(configuration, filePath, keyGeneratorOpt)) {
+ try (ClosableIterator<HoodieKey> iterator = getHoodieKeyIterator(storage, filePath, keyGeneratorOpt)) {
iterator.forEachRemaining(hoodieKeys::add);
return hoodieKeys;
}
@@ -232,15 +231,15 @@ public class ParquetUtils extends FileFormatUtils {
/**
* Get the schema of the given parquet file.
*/
- public MessageType readSchema(StorageConfiguration<?> configuration, StoragePath parquetFilePath) {
- return readMetadata(configuration, parquetFilePath).getFileMetaData().getSchema();
+ public MessageType readSchema(HoodieStorage storage, StoragePath parquetFilePath) {
+ return readMetadata(storage, parquetFilePath).getFileMetaData().getSchema();
}
@Override
- public Map<String, String> readFooter(StorageConfiguration<?> configuration, boolean required,
+ public Map<String, String> readFooter(HoodieStorage storage, boolean required,
StoragePath filePath, String... footerNames) {
Map<String, String> footerVals = new HashMap<>();
- ParquetMetadata footer = readMetadata(configuration, filePath);
+ ParquetMetadata footer = readMetadata(storage, filePath);
Map<String, String> metadata = footer.getFileMetaData().getKeyValueMetaData();
for (String footerName : footerNames) {
if (metadata.containsKey(footerName)) {
@@ -254,16 +253,16 @@ public class ParquetUtils extends FileFormatUtils {
}
@Override
- public Schema readAvroSchema(StorageConfiguration<?> conf, StoragePath filePath) {
- MessageType parquetSchema = readSchema(conf, filePath);
- return new AvroSchemaConverter(conf.unwrapAs(Configuration.class)).convert(parquetSchema);
+ public Schema readAvroSchema(HoodieStorage storage, StoragePath filePath) {
+ MessageType parquetSchema = readSchema(storage, filePath);
+ return new AvroSchemaConverter(storage.getConf().unwrapAs(Configuration.class)).convert(parquetSchema);
}
@Override
- public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(StorageConfiguration<?> storageConf,
+ public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(HoodieStorage storage,
StoragePath filePath,
List<String> columnList) {
- ParquetMetadata metadata = readMetadata(storageConf, filePath);
+ ParquetMetadata metadata = readMetadata(storage, filePath);
// NOTE: This collector has to have fully specialized generic type params since
// Java 1.8 struggles to infer them
@@ -317,10 +316,10 @@ public class ParquetUtils extends FileFormatUtils {
* NOTE: This literally reads the entire file contents, thus should be used with caution.
*/
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath) {
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath) {
List<GenericRecord> records = new ArrayList<>();
try (ParquetReader reader = AvroParquetReader.builder(new Path(filePath.toUri()))
- .withConf(configuration.unwrapAs(Configuration.class)).build()) {
+ .withConf(storage.getConf().unwrapAs(Configuration.class)).build()) {
Object obj = reader.read();
while (obj != null) {
if (obj instanceof GenericRecord) {
@@ -336,22 +335,22 @@ public class ParquetUtils extends FileFormatUtils {
}
@Override
- public List<GenericRecord> readAvroRecords(StorageConfiguration<?> configuration, StoragePath filePath, Schema schema) {
- AvroReadSupport.setAvroReadSchema(configuration.unwrapAs(Configuration.class), schema);
- return readAvroRecords(configuration, filePath);
+ public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath, Schema schema) {
+ AvroReadSupport.setAvroReadSchema(storage.getConf().unwrapAs(Configuration.class), schema);
+ return readAvroRecords(storage, filePath);
}
/**
* Returns the number of records in the parquet file.
*
- * @param conf Configuration
+ * @param storage {@link HoodieStorage} instance.
* @param filePath path of the file
*/
@Override
- public long getRowCount(StorageConfiguration<?> conf, StoragePath filePath) {
+ public long getRowCount(HoodieStorage storage, StoragePath filePath) {
ParquetMetadata footer;
long rowCount = 0;
- footer = readMetadata(conf, filePath);
+ footer = readMetadata(storage, filePath);
for (BlockMetaData b : footer.getBlocks()) {
rowCount += b.getRowCount();
}
@@ -375,7 +374,7 @@ public class ParquetUtils extends FileFormatUtils {
}
@Override
- public byte[] serializeRecordsToLogBlock(StorageConfiguration<?> storageConf,
+ public byte[] serializeRecordsToLogBlock(HoodieStorage storage,
List<HoodieRecord> records,
Schema writerSchema,
Schema readerSchema,
@@ -395,7 +394,7 @@ public class ParquetUtils extends FileFormatUtils {
HoodieFileWriter parquetWriter = null;
try {
parquetWriter = HoodieFileWriterFactory.getFileWriter(
- HoodieFileFormat.PARQUET, outputStream, storageConf, config, writerSchema, recordType);
+ HoodieFileFormat.PARQUET, outputStream, storage, config, writerSchema, recordType);
for (HoodieRecord<?> record : records) {
String recordKey = record.getRecordKey(readerSchema, keyFieldName);
parquetWriter.write(recordKey, record, writerSchema);
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java
index d3a340adfbb..1c455240be1 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java
@@ -26,7 +26,6 @@ import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.io.storage.HoodieNativeAvroHFileReader;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -35,13 +34,13 @@ import java.io.IOException;
public class HoodieAvroFileReaderFactory extends HoodieFileReaderFactory {
- public HoodieAvroFileReaderFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieAvroFileReaderFactory(HoodieStorage storage) {
+ super(storage);
}
@Override
protected HoodieFileReader newParquetFileReader(StoragePath path) {
- return new HoodieAvroParquetReader(storageConf, path);
+ return new HoodieAvroParquetReader(storage, path);
}
@Override
@@ -49,12 +48,12 @@ public class HoodieAvroFileReaderFactory extends HoodieFileReaderFactory {
StoragePath path,
Option<Schema> schemaOption) throws IOException {
if (isUseNativeHFileReaderEnabled(hoodieConfig)) {
- return new HoodieNativeAvroHFileReader(storageConf, path, schemaOption);
+ return new HoodieNativeAvroHFileReader(storage, path, schemaOption);
}
if (schemaOption.isPresent()) {
- return new HoodieHBaseAvroHFileReader(storageConf, path, schemaOption);
+ return new HoodieHBaseAvroHFileReader(storage.getConf(), path, schemaOption);
}
- return new HoodieHBaseAvroHFileReader(storageConf, path);
+ return new HoodieHBaseAvroHFileReader(storage.getConf(), path);
}
@Override
@@ -64,14 +63,14 @@ public class HoodieAvroFileReaderFactory extends HoodieFileReaderFactory {
byte[] content,
Option<Schema> schemaOption) throws IOException {
if (isUseNativeHFileReaderEnabled(hoodieConfig)) {
- return new HoodieNativeAvroHFileReader(storageConf, content, schemaOption);
+ return new HoodieNativeAvroHFileReader(this.storage, content, schemaOption);
}
- return new HoodieHBaseAvroHFileReader(storageConf, path, storage, content, schemaOption);
+ return new HoodieHBaseAvroHFileReader(this.storage.getConf(), path, storage, content, schemaOption);
}
@Override
protected HoodieFileReader newOrcFileReader(StoragePath path) {
- return new HoodieAvroOrcReader(storageConf, path);
+ return new HoodieAvroOrcReader(storage, path);
}
@Override
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java
index 0ce60074c2d..aaf583afefb 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java
@@ -32,7 +32,7 @@ import org.apache.hudi.io.storage.HoodieFileWriter;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.io.storage.HoodieOrcConfig;
import org.apache.hudi.io.storage.HoodieParquetConfig;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -55,8 +55,8 @@ import static org.apache.hudi.io.hadoop.HoodieHFileConfig.PREFETCH_ON_OPEN;
public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
- public HoodieAvroFileWriterFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieAvroFileWriterFactory(HoodieStorage storage) {
+ super(storage);
}
@Override
@@ -76,7 +76,7 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
config.getIntOrDefault(HoodieStorageConfig.PARQUET_BLOCK_SIZE),
config.getIntOrDefault(HoodieStorageConfig.PARQUET_PAGE_SIZE),
config.getLongOrDefault(HoodieStorageConfig.PARQUET_MAX_FILE_SIZE),
- storageConf, config.getDoubleOrDefault(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION),
+ storage.getConf(), config.getDoubleOrDefault(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION),
config.getBooleanOrDefault(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED));
return new HoodieAvroParquetWriter(path, parquetConfig, instantTime, taskContextSupplier, populateMetaFields);
}
@@ -89,7 +89,7 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
config.getInt(HoodieStorageConfig.PARQUET_BLOCK_SIZE),
config.getInt(HoodieStorageConfig.PARQUET_PAGE_SIZE),
config.getLong(HoodieStorageConfig.PARQUET_MAX_FILE_SIZE), // todo: 1024*1024*1024
- storageConf, config.getDouble(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION),
+ storage.getConf(), config.getDouble(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION),
config.getBoolean(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED));
return new HoodieParquetStreamWriter(new FSDataOutputStream(outputStream, null), parquetConfig);
}
@@ -98,7 +98,7 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
String instantTime, StoragePath path, HoodieConfig config, Schema schema,
TaskContextSupplier taskContextSupplier) throws IOException {
BloomFilter filter = createBloomFilter(config);
- HoodieHFileConfig hfileConfig = new HoodieHFileConfig(storageConf.unwrapAs(Configuration.class),
+ HoodieHFileConfig hfileConfig = new HoodieHFileConfig(storage.getConf().unwrapAs(Configuration.class),
Compression.Algorithm.valueOf(
config.getString(HoodieStorageConfig.HFILE_COMPRESSION_ALGORITHM_NAME)),
config.getInt(HoodieStorageConfig.HFILE_BLOCK_SIZE),
@@ -112,7 +112,7 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
String instantTime, StoragePath path, HoodieConfig config, Schema schema,
TaskContextSupplier taskContextSupplier) throws IOException {
BloomFilter filter = createBloomFilter(config);
- HoodieOrcConfig orcConfig = new HoodieOrcConfig(storageConf,
+ HoodieOrcConfig orcConfig = new HoodieOrcConfig(storage.getConf(),
CompressionKind.valueOf(config.getString(HoodieStorageConfig.ORC_COMPRESSION_CODEC_NAME)),
config.getInt(HoodieStorageConfig.ORC_STRIPE_SIZE),
config.getInt(HoodieStorageConfig.ORC_BLOCK_SIZE),
@@ -126,6 +126,6 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
return (HoodieAvroWriteSupport) ReflectionUtils.loadClass(
config.getStringOrDefault(HoodieStorageConfig.HOODIE_AVRO_WRITE_SUPPORT_CLASS),
new Class<?>[] {MessageType.class, Schema.class, Option.class, Properties.class},
- new AvroSchemaConverter(storageConf.unwrapAs(Configuration.class)).convert(schema), schema, filter, config.getProps());
+ new AvroSchemaConverter(storage.getConf().unwrapAs(Configuration.class)).convert(schema), schema, filter, config.getProps());
}
}
\ No newline at end of file
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
index 9f8b453535b..c709c5ef4f4 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
@@ -27,7 +27,7 @@ import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieFileReader;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -51,28 +51,28 @@ import java.util.Set;
public class HoodieAvroOrcReader extends HoodieAvroFileReader {
private final StoragePath path;
- private final StorageConfiguration<?> conf;
+ private final HoodieStorage storage;
private final FileFormatUtils orcUtils;
- public HoodieAvroOrcReader(StorageConfiguration<?> configuration, StoragePath path) {
- this.conf = configuration;
+ public HoodieAvroOrcReader(HoodieStorage storage, StoragePath path) {
+ this.storage = storage;
this.path = path;
this.orcUtils = FileFormatUtils.getInstance(HoodieFileFormat.ORC);
}
@Override
public String[] readMinMaxRecordKeys() {
- return orcUtils.readMinMaxRecordKeys(conf, path);
+ return orcUtils.readMinMaxRecordKeys(storage, path);
}
@Override
public BloomFilter readBloomFilter() {
- return orcUtils.readBloomFilterFromMetadata(conf, path);
+ return orcUtils.readBloomFilterFromMetadata(storage, path);
}
@Override
public Set<String> filterRowKeys(Set candidateRowKeys) {
- return orcUtils.filterRowKeys(conf, path, candidateRowKeys);
+ return orcUtils.filterRowKeys(storage, path, candidateRowKeys);
}
@Override
@@ -81,7 +81,7 @@ public class HoodieAvroOrcReader extends HoodieAvroFileReader {
throw new UnsupportedOperationException("Schema projections are not supported in HFile reader");
}
- Configuration hadoopConf = conf.unwrapAs(Configuration.class);
+ Configuration hadoopConf = storage.getConf().unwrapAs(Configuration.class);
try (Reader reader = OrcFile.createReader(new Path(path.toUri()), OrcFile.readerOptions(hadoopConf))) {
TypeDescription orcSchema = AvroOrcUtils.createOrcSchema(readerSchema);
RecordReader recordReader = reader.rows(new Options(hadoopConf).schema(orcSchema));
@@ -93,7 +93,7 @@ public class HoodieAvroOrcReader extends HoodieAvroFileReader {
@Override
public ClosableIterator<String> getRecordKeyIterator() {
- final Iterator<String> iterator = orcUtils.readRowKeys(conf, path).iterator();
+ final Iterator<String> iterator = orcUtils.readRowKeys(storage, path).iterator();
return new ClosableIterator<String>() {
@Override
public boolean hasNext() {
@@ -113,7 +113,7 @@ public class HoodieAvroOrcReader extends HoodieAvroFileReader {
@Override
public Schema getSchema() {
- return orcUtils.readAvroSchema(conf, path);
+ return orcUtils.readAvroSchema(storage, path);
}
@Override
@@ -122,6 +122,6 @@ public class HoodieAvroOrcReader extends HoodieAvroFileReader {
@Override
public long getTotalRecords() {
- return orcUtils.getRowCount(conf, path);
+ return orcUtils.getRowCount(storage, path);
}
}
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
index 76614dfea95..22af48fc7b7 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
@@ -31,6 +31,7 @@ import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.common.util.collection.CloseableMappingIterator;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieFileReader;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
@@ -57,14 +58,14 @@ import static org.apache.hudi.common.util.TypeUtils.unsafeCast;
public class HoodieAvroParquetReader extends HoodieAvroFileReader {
private final StoragePath path;
- private final StorageConfiguration<?> conf;
+ private final HoodieStorage storage;
private final FileFormatUtils parquetUtils;
private final List<ParquetReaderIterator> readerIterators = new ArrayList<>();
- public HoodieAvroParquetReader(StorageConfiguration<?> storageConf, StoragePath path) {
+ public HoodieAvroParquetReader(HoodieStorage storage, StoragePath path) {
// We have to clone the Hadoop Config as it might be subsequently modified
// by the Reader (for proper config propagation to Parquet components)
- this.conf = tryOverrideDefaultConfigs(storageConf.newInstance());
+ this.storage = storage.newInstance(path, tryOverrideDefaultConfigs(storage.getConf().newInstance()));
this.path = path;
this.parquetUtils = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET);
}
@@ -80,17 +81,17 @@ public class HoodieAvroParquetReader extends HoodieAvroFileReader {
@Override
public String[] readMinMaxRecordKeys() {
- return parquetUtils.readMinMaxRecordKeys(conf, path);
+ return parquetUtils.readMinMaxRecordKeys(storage, path);
}
@Override
public BloomFilter readBloomFilter() {
- return parquetUtils.readBloomFilterFromMetadata(conf, path);
+ return parquetUtils.readBloomFilterFromMetadata(storage, path);
}
@Override
public Set<String> filterRowKeys(Set<String> candidateRowKeys) {
- return parquetUtils.filterRowKeys(conf, path, candidateRowKeys);
+ return parquetUtils.filterRowKeys(storage, path, candidateRowKeys);
}
@Override
@@ -105,7 +106,7 @@ public class HoodieAvroParquetReader extends HoodieAvroFileReader {
@Override
public Schema getSchema() {
- return parquetUtils.readAvroSchema(conf, path);
+ return parquetUtils.readAvroSchema(storage, path);
}
@Override
@@ -115,7 +116,7 @@ public class HoodieAvroParquetReader extends HoodieAvroFileReader {
@Override
public long getTotalRecords() {
- return parquetUtils.getRowCount(conf, path);
+ return parquetUtils.getRowCount(storage, path);
}
private static StorageConfiguration<?> tryOverrideDefaultConfigs(StorageConfiguration<?> conf) {
@@ -162,7 +163,7 @@ public class HoodieAvroParquetReader extends HoodieAvroFileReader {
// NOTE: We have to set both Avro read-schema and projection schema to make
// sure that in case the file-schema is not equal to read-schema we'd still
// be able to read that file (in case projection is a proper one)
- Configuration hadoopConf = conf.unwrapAs(Configuration.class);
+ Configuration hadoopConf = storage.getConf().unwrapAs(Configuration.class);
if (!requestedSchema.isPresent()) {
AvroReadSupport.setAvroReadSchema(hadoopConf, schema);
AvroReadSupport.setRequestedProjection(hadoopConf, schema);
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieHadoopIOFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHadoopIOFactory.java
similarity index 81%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieHadoopIOFactory.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHadoopIOFactory.java
index c357a70be3e..4203fe90b4b 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieHadoopIOFactory.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHadoopIOFactory.java
@@ -17,16 +17,16 @@
* under the License.
*/
-package org.apache.hudi.io.storage;
+package org.apache.hudi.io.hadoop;
import org.apache.hudi.common.fs.ConsistencyGuard;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.io.hadoop.HoodieAvroFileReaderFactory;
-import org.apache.hudi.io.hadoop.HoodieAvroFileWriterFactory;
+import org.apache.hudi.io.storage.HoodieFileReaderFactory;
+import org.apache.hudi.io.storage.HoodieFileWriterFactory;
+import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
@@ -37,21 +37,21 @@ import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
*/
public class HoodieHadoopIOFactory extends HoodieIOFactory {
- public HoodieHadoopIOFactory(StorageConfiguration<?> storageConf) {
- super(storageConf);
+ public HoodieHadoopIOFactory(HoodieStorage storage) {
+ super(storage);
}
@Override
public HoodieFileReaderFactory getReaderFactory(HoodieRecord.HoodieRecordType recordType) {
switch (recordType) {
case AVRO:
- return new HoodieAvroFileReaderFactory(storageConf);
+ return new HoodieAvroFileReaderFactory(storage);
case SPARK:
//TODO: remove this case [HUDI-7746]
try {
return (HoodieFileReaderFactory) ReflectionUtils
.loadClass("org.apache.hudi.io.storage.HoodieSparkFileReaderFactory",
- new Class<?>[] {StorageConfiguration.class}, storageConf);
+ new Class<?>[] {HoodieStorage.class}, storage);
} catch (Exception e) {
throw new HoodieException("Unable to create HoodieSparkFileReaderFactory", e);
}
@@ -64,13 +64,13 @@ public class HoodieHadoopIOFactory extends HoodieIOFactory {
public HoodieFileWriterFactory getWriterFactory(HoodieRecord.HoodieRecordType recordType) {
switch (recordType) {
case AVRO:
- return new HoodieAvroFileWriterFactory(storageConf);
+ return new HoodieAvroFileWriterFactory(storage);
case SPARK:
//TODO: remove this case [HUDI-7746]
try {
return (HoodieFileWriterFactory) ReflectionUtils
.loadClass("org.apache.hudi.io.storage.HoodieSparkFileWriterFactory",
- new Class<?>[] {StorageConfiguration.class}, storageConf);
+ new Class<?>[] {HoodieStorage.class}, storage);
} catch (Exception e) {
throw new HoodieException("Unable to create HoodieSparkFileWriterFactory", e);
}
@@ -81,7 +81,7 @@ public class HoodieHadoopIOFactory extends HoodieIOFactory {
@Override
public HoodieStorage getStorage(StoragePath storagePath) {
- return new HoodieHadoopStorage(storagePath, storageConf);
+ return storage.newInstance(storagePath, storage.getConf());
}
@Override
@@ -92,7 +92,7 @@ public class HoodieHadoopIOFactory extends HoodieIOFactory {
long initialRetryIntervalMs,
String retryExceptions,
ConsistencyGuard consistencyGuard) {
- return new HoodieHadoopStorage(path, storageConf, enableRetry, maxRetryIntervalMs,
+ return new HoodieHadoopStorage(path, storage.getConf(), enableRetry, maxRetryIntervalMs,
maxRetryNumbers, maxRetryIntervalMs, retryExceptions, consistencyGuard);
}
}
\ No newline at end of file
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
index f7df5e1f164..10826c7aace 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
@@ -59,7 +59,23 @@ public class HoodieHadoopStorage extends HoodieStorage {
private final FileSystem fs;
public HoodieHadoopStorage(StoragePath path, StorageConfiguration<?> conf) {
- this(HadoopFSUtils.getFs(path, conf.unwrapAs(Configuration.class)));
+ super(conf);
+ this.fs = HadoopFSUtils.getFs(path, conf.unwrapAs(Configuration.class));
+ }
+
+ public HoodieHadoopStorage(Path path, Configuration conf) {
+ super(HadoopFSUtils.getStorageConf(conf));
+ this.fs = HadoopFSUtils.getFs(path, conf);
+ }
+
+ public HoodieHadoopStorage(String path, Configuration conf) {
+ super(HadoopFSUtils.getStorageConf(conf));
+ this.fs = HadoopFSUtils.getFs(path, conf);
+ }
+
+ public HoodieHadoopStorage(String path, StorageConfiguration<?> conf) {
+ super(conf);
+ this.fs = HadoopFSUtils.getFs(path, conf);
}
public HoodieHadoopStorage(StoragePath path,
@@ -70,6 +86,7 @@ public class HoodieHadoopStorage extends HoodieStorage {
long initialRetryIntervalMs,
String retryExceptions,
ConsistencyGuard consistencyGuard) {
+ super(conf);
FileSystem fileSystem = getFs(path, conf.unwrapCopyAs(Configuration.class));
if (enableRetry) {
@@ -82,9 +99,15 @@ public class HoodieHadoopStorage extends HoodieStorage {
}
public HoodieHadoopStorage(FileSystem fs) {
+ super(new HadoopStorageConfiguration(fs.getConf()));
this.fs = fs;
}
+ @Override
+ public HoodieStorage newInstance(StoragePath path, StorageConfiguration<?> storageConf) {
+ return new HoodieHadoopStorage(path, storageConf);
+ }
+
@Override
public String getScheme() {
return fs.getScheme();
@@ -225,11 +248,6 @@ public class HoodieHadoopStorage extends HoodieStorage {
return fs;
}
- @Override
- public StorageConfiguration<Configuration> getConf() {
- return new HadoopStorageConfiguration(fs.getConf());
- }
-
@Override
public HoodieStorage getRawStorage() {
if (fs instanceof HoodieWrapperFileSystem) {
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
index f7a98a4b2fe..7c282536423 100755
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
@@ -432,9 +432,9 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString());
- byte[] dataBlockContentBytes = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records, header).getContentBytes(storage.getConf());
+ byte[] dataBlockContentBytes = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records, header).getContentBytes(storage);
HoodieLogBlock.HoodieLogBlockContentLocation logBlockContentLoc = new HoodieLogBlock.HoodieLogBlockContentLocation(
- HoodieTestUtils.getDefaultStorageConfWithDefaults(), null, 0, dataBlockContentBytes.length, 0);
+ HoodieTestUtils.getStorage(basePath), null, 0, dataBlockContentBytes.length, 0);
HoodieDataBlock reusableDataBlock = new HoodieAvroDataBlock(null, Option.ofNullable(dataBlockContentBytes), false,
logBlockContentLoc, Option.ofNullable(getSimpleSchema()), header, new HashMap<>(), HoodieRecord.RECORD_KEY_METADATA_FIELD);
long writtenSize = 0;
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
index 86f6640caf0..f7ef520c38e 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
@@ -29,8 +29,8 @@ import org.apache.hudi.common.testutils.SchemaTestUtil;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.internal.schema.HoodieSchemaException;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
@@ -95,7 +95,7 @@ public class TestTableSchemaResolver {
StoragePath partitionPath = new StoragePath(testDir, "partition1");
Schema expectedSchema = getSimpleSchema();
StoragePath logFilePath = writeLogFile(partitionPath, expectedSchema);
- assertEquals(expectedSchema, TableSchemaResolver.readSchemaFromLogFile(HoodieStorageUtils.getStorage(
+ assertEquals(expectedSchema, TableSchemaResolver.readSchemaFromLogFile(new HoodieHadoopStorage(
logFilePath, HoodieTestUtils.getDefaultStorageConfWithDefaults()), logFilePath));
}
@@ -106,8 +106,7 @@ public class TestTableSchemaResolver {
}
private StoragePath writeLogFile(StoragePath partitionPath, Schema schema) throws IOException, URISyntaxException, InterruptedException {
- HoodieStorage storage = HoodieStorageUtils.getStorage(
- partitionPath, HoodieTestUtils.getDefaultStorageConfWithDefaults());
+ HoodieStorage storage = HoodieTestUtils.getStorage(partitionPath);
HoodieLogFormat.Writer writer =
HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath).withFileExtension(HoodieLogFile.DELTA_EXTENSION)
.withFileId("test-fileid1").overBaseCommit("100").withStorage(storage).build();
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieDeleteBlock.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieDeleteBlock.java
index 2e46b93d4b5..700ba2f5137 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieDeleteBlock.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieDeleteBlock.java
@@ -118,7 +118,7 @@ public class TestHoodieDeleteBlock {
public void testDeleteBlockWithValidation(DeleteRecord[] deleteRecords) throws IOException {
HoodieDeleteBlock deleteBlock = new HoodieDeleteBlock(deleteRecords, new HashMap<>());
- byte[] contentBytes = deleteBlock.getContentBytes(HoodieTestUtils.getDefaultStorageConf());
+ byte[] contentBytes = deleteBlock.getContentBytes(HoodieTestUtils.getDefaultStorage());
HoodieDeleteBlock deserializeDeleteBlock = new HoodieDeleteBlock(
Option.of(contentBytes), null, true, Option.empty(), new HashMap<>(), new HashMap<>());
DeleteRecord[] deserializedDeleteRecords = deserializeDeleteBlock.getRecordsToDelete();
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
index f6caa31a62c..e4f316170a9 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
@@ -24,11 +24,8 @@ import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.minicluster.HdfsTestService;
import org.apache.hudi.exception.HoodieIOException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -188,8 +185,7 @@ public class TestDFSPropertiesConfiguration {
ENVIRONMENT_VARIABLES.clear(DFSPropertiesConfiguration.CONF_FILE_DIR_ENV_NAME);
DFSPropertiesConfiguration.refreshGlobalProps();
try {
- if (!HoodieStorageUtils.getStorage(
- DFSPropertiesConfiguration.DEFAULT_PATH, HadoopFSUtils.getStorageConf(new Configuration()))
+ if (!HoodieTestUtils.getStorage(DFSPropertiesConfiguration.DEFAULT_PATH)
.exists(DFSPropertiesConfiguration.DEFAULT_PATH)) {
assertEquals(0, DFSPropertiesConfiguration.getGlobalProps().size());
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
index 086cf70c4a7..55a693162bb 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
@@ -98,13 +98,13 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
// Read and verify
List<String> rowKeysInFile = new ArrayList<>(parquetUtils.readRowKeys(
- HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath)));
+ HoodieTestUtils.getStorage(filePath), new StoragePath(filePath)));
Collections.sort(rowKeysInFile);
Collections.sort(rowKeys);
assertEquals(rowKeys, rowKeysInFile, "Did not read back the expected list of keys");
BloomFilter filterInFile = parquetUtils.readBloomFilterFromMetadata(
- HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath));
+ HoodieTestUtils.getStorage(filePath), new StoragePath(filePath));
for (String rowKey : rowKeys) {
assertTrue(filterInFile.mightContain(rowKey), "key should be found in bloom filter");
}
@@ -128,7 +128,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
// Read and verify
Set<String> filtered =
- parquetUtils.filterRowKeys(HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath), filter);
+ parquetUtils.filterRowKeys(HoodieTestUtils.getStorage(filePath), new StoragePath(filePath), filter);
assertEquals(filter.size(), filtered.size(), "Filtered count does not match");
@@ -155,7 +155,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
// Read and verify
List<HoodieKey> fetchedRows =
- parquetUtils.fetchHoodieKeys(HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath));
+ parquetUtils.fetchHoodieKeys(HoodieTestUtils.getStorage(filePath), new StoragePath(filePath));
assertEquals(rowKeys.size(), fetchedRows.size(), "Total count does not match");
for (HoodieKey entry : fetchedRows) {
@@ -181,7 +181,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
// Read and verify
List<HoodieKey> fetchedRows =
- parquetUtils.fetchHoodieKeys(HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath),
+ parquetUtils.fetchHoodieKeys(HoodieTestUtils.getStorage(filePath), new StoragePath(filePath),
Option.of(new TestBaseKeyGen("abc","def")));
assertEquals(rowKeys.size(), fetchedRows.size(), "Total count does not match");
@@ -200,7 +200,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
writeParquetFile(BloomFilterTypeCode.SIMPLE.name(), filePath, rowKeys);
assertEquals(123, parquetUtils.getRowCount(
- HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath)));
+ HoodieTestUtils.getStorage(filePath), new StoragePath(filePath)));
}
@Test
@@ -266,7 +266,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
columnList.add(dataField);
List<HoodieColumnRangeMetadata<Comparable>> columnRangeMetadataList = parquetUtils.readColumnStatsFromMetadata(
- HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath), columnList)
+ HoodieTestUtils.getStorage(filePath), new StoragePath(filePath), columnList)
.stream()
.sorted(Comparator.comparing(HoodieColumnRangeMetadata::getColumnName))
.collect(Collectors.toList());
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java
index b46e0747aeb..1bb2324505a 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java
@@ -20,14 +20,13 @@
package org.apache.hudi.io.hadoop;
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.io.storage.HoodieIOFactory;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
-import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -47,9 +46,9 @@ public class TestHoodieAvroFileReaderFactory {
@Test
public void testGetFileReader() throws IOException {
// parquet file format.
- final StorageConfiguration<?> storageConf = HadoopFSUtils.getStorageConf(new Configuration());
+ final HoodieStorage storage = HoodieTestUtils.getDefaultStorage();
final StoragePath parquetPath = new StoragePath("/partition/path/f1_1-0-1_000.parquet");
- HoodieFileReader parquetReader = HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecordType.AVRO)
+ HoodieFileReader parquetReader = HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, parquetPath);
assertTrue(parquetReader instanceof HoodieAvroParquetReader);
@@ -57,14 +56,14 @@ public class TestHoodieAvroFileReaderFactory {
final StoragePath logPath = new StoragePath(
"/partition/path/f.b51192a8-574b-4a85-b246-bcfec03ac8bf_100.log.2_1-0-1");
final Throwable thrown = assertThrows(UnsupportedOperationException.class, () -> {
- HoodieFileReader logWriter = HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecordType.AVRO)
+ HoodieFileReader logWriter = HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, logPath);
}, "should fail since log storage reader is not supported yet.");
assertTrue(thrown.getMessage().contains("format not supported yet."));
// Orc file format.
final StoragePath orcPath = new StoragePath("/partition/path/f1_1-0-1_000.orc");
- HoodieFileReader orcReader = HoodieIOFactory.getIOFactory(storageConf)
+ HoodieFileReader orcReader = HoodieIOFactory.getIOFactory(storage)
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, orcPath);
assertTrue(orcReader instanceof HoodieAvroOrcReader);
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHBaseHFileReaderWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHBaseHFileReaderWriter.java
index f48b9aeffa9..f34034d0a35 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHBaseHFileReaderWriter.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHBaseHFileReaderWriter.java
@@ -19,13 +19,12 @@
package org.apache.hudi.io.hadoop;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -60,16 +59,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
public class TestHoodieHBaseHFileReaderWriter extends TestHoodieHFileReaderWriterBase {
@Override
protected HoodieAvroFileReader createReader(
- StorageConfiguration<?> conf) throws Exception {
- return new HoodieHBaseAvroHFileReader(conf, getFilePath(), Option.empty());
+ HoodieStorage storage) throws Exception {
+ return new HoodieHBaseAvroHFileReader(storage.getConf(), getFilePath(), Option.empty());
}
@Override
- protected HoodieAvroHFileReaderImplBase createHFileReader(StorageConfiguration<?> conf,
+ protected HoodieAvroHFileReaderImplBase createHFileReader(HoodieStorage storage,
byte[] content) throws IOException {
FileSystem fs = HadoopFSUtils.getFs(getFilePath().toString(), new Configuration());
- return new HoodieHBaseAvroHFileReader(conf, new StoragePath(DUMMY_BASE_PATH),
- HoodieStorageUtils.getStorage(getFilePath(), conf), content, Option.empty());
+ return new HoodieHBaseAvroHFileReader(storage.getConf(), new StoragePath(DUMMY_BASE_PATH), storage, content, Option.empty());
}
@Override
@@ -78,8 +76,7 @@ public class TestHoodieHBaseHFileReaderWriter extends TestHoodieHFileReaderWrite
boolean mayUseDefaultComparator,
Class<?> expectedComparatorClazz,
int count) throws IOException {
- HoodieStorage storage = HoodieStorageUtils.getStorage(
- getFilePath(), HadoopFSUtils.getStorageConf(new Configuration()));
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
try (HFile.Reader reader =
HoodieHFileUtils.createHFileReader(storage, new StoragePath(DUMMY_BASE_PATH), content)) {
// HFile version is 3
@@ -99,7 +96,7 @@ public class TestHoodieHBaseHFileReaderWriter extends TestHoodieHFileReaderWrite
public void testReaderGetRecordIteratorByKeysWithBackwardSeek() throws Exception {
writeFileWithSimpleSchema();
try (HoodieAvroHFileReaderImplBase hfileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
List<GenericRecord> allRecords = toStream(hfileReader.getRecordIterator())
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
index b87af2c8371..362f58a00cf 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
@@ -19,16 +19,15 @@
package org.apache.hudi.io.hadoop;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieNativeAvroHFileReader;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.Test;
import java.io.IOException;
@@ -47,14 +46,14 @@ public class TestHoodieHFileReaderWriter extends TestHoodieHFileReaderWriterBase
@Override
protected HoodieAvroFileReader createReader(
- StorageConfiguration<?> conf) throws Exception {
- return new HoodieNativeAvroHFileReader(conf, getFilePath(), Option.empty());
+ HoodieStorage storage) throws Exception {
+ return new HoodieNativeAvroHFileReader(storage, getFilePath(), Option.empty());
}
@Override
- protected HoodieAvroHFileReaderImplBase createHFileReader(StorageConfiguration<?> conf,
+ protected HoodieAvroHFileReaderImplBase createHFileReader(HoodieStorage storage,
byte[] content) throws IOException {
- return new HoodieNativeAvroHFileReader(conf, content, Option.empty());
+ return new HoodieNativeAvroHFileReader(storage, content, Option.empty());
}
@Override
@@ -63,8 +62,7 @@ public class TestHoodieHFileReaderWriter extends TestHoodieHFileReaderWriterBase
boolean mayUseDefaultComparator,
Class<?> expectedComparatorClazz,
int count) throws IOException {
- try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(
- HadoopFSUtils.getStorageConf(new Configuration()), content)) {
+ try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(HoodieTestUtils.getStorage(hfileName), content)) {
assertEquals(count, hfileReader.getTotalRecords());
}
}
@@ -73,7 +71,7 @@ public class TestHoodieHFileReaderWriter extends TestHoodieHFileReaderWriterBase
public void testReaderGetRecordIteratorByKeysWithBackwardSeek() throws Exception {
writeFileWithSimpleSchema();
try (HoodieAvroHFileReaderImplBase hfileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
// Filter for "key00001, key05, key24, key16, key31, key61".
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriterBase.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriterBase.java
index 1d69115315a..3ed3343e93c 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriterBase.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriterBase.java
@@ -27,14 +27,13 @@ import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
@@ -74,11 +73,11 @@ import java.util.stream.StreamSupport;
import static org.apache.hudi.common.testutils.FileSystemTestUtils.RANDOM;
import static org.apache.hudi.common.testutils.SchemaTestUtil.getSchemaFromResource;
import static org.apache.hudi.common.util.CollectionUtils.toStream;
+import static org.apache.hudi.io.hadoop.HoodieHFileConfig.HFILE_COMPARATOR;
import static org.apache.hudi.io.hfile.TestHFileReader.BOOTSTRAP_INDEX_HFILE_SUFFIX;
import static org.apache.hudi.io.hfile.TestHFileReader.COMPLEX_SCHEMA_HFILE_SUFFIX;
import static org.apache.hudi.io.hfile.TestHFileReader.SIMPLE_SCHEMA_HFILE_SUFFIX;
import static org.apache.hudi.io.hfile.TestHFileReader.readHFileFromResources;
-import static org.apache.hudi.io.hadoop.HoodieHFileConfig.HFILE_COMPARATOR;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -92,7 +91,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
// Number of records in HFile fixtures for compatibility tests
protected static final int NUM_RECORDS_FIXTURE = 50;
- protected abstract HoodieAvroHFileReaderImplBase createHFileReader(StorageConfiguration<?> conf,
+ protected abstract HoodieAvroHFileReaderImplBase createHFileReader(HoodieStorage storage,
byte[] content) throws IOException;
protected abstract void verifyHFileReader(byte[] content,
@@ -114,7 +113,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
protected HoodieAvroHFileWriter createWriter(
Schema avroSchema, boolean populateMetaFields) throws Exception {
String instantTime = "000";
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
Properties props = new Properties();
props.setProperty(HoodieTableConfig.POPULATE_META_FIELDS.key(), Boolean.toString(populateMetaFields));
TaskContextSupplier mockTaskContextSupplier = Mockito.mock(TaskContextSupplier.class);
@@ -123,7 +122,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
when(partitionSupplier.get()).thenReturn(10);
return (HoodieAvroHFileWriter) HoodieFileWriterFactory.getFileWriter(
- instantTime, getFilePath(), conf, HoodieStorageConfig.newBuilder().fromProperties(props).build(), avroSchema,
+ instantTime, getFilePath(), storage, HoodieStorageConfig.newBuilder().fromProperties(props).build(), avroSchema,
mockTaskContextSupplier, HoodieRecord.HoodieRecordType.AVRO);
}
@@ -133,8 +132,8 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
}
@Override
- protected void verifyMetadata(StorageConfiguration<?> conf) throws IOException {
- try (HoodieAvroFileReader reader = createReader(conf)) {
+ protected void verifyMetadata(HoodieStorage storage) throws IOException {
+ try (HoodieAvroFileReader reader = createReader(storage)) {
assertEquals(NUM_RECORDS, reader.getTotalRecords());
} catch (Exception e) {
throw new RuntimeException(e);
@@ -142,8 +141,8 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
}
@Override
- protected void verifySchema(StorageConfiguration<?> conf, String schemaPath) throws IOException {
- try (HoodieAvroFileReader reader = createReader(conf)) {
+ protected void verifySchema(HoodieStorage storage, String schemaPath) throws IOException {
+ try (HoodieAvroFileReader reader = createReader(storage)) {
assertEquals(
getSchemaFromResource(TestHoodieHBaseHFileReaderWriter.class, schemaPath),
reader.getSchema());
@@ -180,9 +179,9 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
}
writer.close();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
HoodieAvroHFileReaderImplBase hoodieHFileReader =
- (HoodieAvroHFileReaderImplBase) createReader(conf);
+ (HoodieAvroHFileReaderImplBase) createReader(storage);
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
assertEquals(new ArrayList<>(recordMap.values()), records);
@@ -198,7 +197,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
List<GenericRecord> expectedRecords =
rowsList.stream().map(recordMap::get).collect(Collectors.toList());
- hoodieHFileReader = (HoodieAvroHFileReaderImplBase) createReader(conf);
+ hoodieHFileReader = (HoodieAvroHFileReaderImplBase) createReader(storage);
List<GenericRecord> result =
HoodieAvroHFileReaderImplBase.readRecords(hoodieHFileReader, rowsList).stream()
.map(r -> (GenericRecord) r).collect(Collectors.toList());
@@ -226,12 +225,11 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
@Test
public void testReadHFileFormatRecords() throws Exception {
writeFileWithSimpleSchema();
- HoodieStorage storage = HoodieStorageUtils.getStorage(getFilePath(), HadoopFSUtils.getStorageConf(new Configuration()));
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
byte[] content = FileIOUtils.readAsByteArray(
storage.open(getFilePath()), (int) storage.getPathInfo(getFilePath()).getLength());
// Reading byte array in HFile format, without actual file path
- StorageConfiguration<?> storageConf = storage.getConf();
- try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storageConf, content)) {
+ try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storage, content)) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
assertEquals(NUM_RECORDS, hfileReader.getTotalRecords());
@@ -243,7 +241,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
public void testReaderGetRecordIterator() throws Exception {
writeFileWithSimpleSchema();
try (HoodieAvroHFileReaderImplBase hfileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
List<String> keys =
IntStream.concat(IntStream.range(40, NUM_RECORDS * 2), IntStream.range(10, 20))
.mapToObj(i -> "key" + String.format("%02d", i)).collect(Collectors.toList());
@@ -271,7 +269,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
public void testReaderGetRecordIteratorByKeys() throws Exception {
writeFileWithSimpleSchema();
try (HoodieAvroHFileReaderImplBase hfileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
@@ -315,7 +313,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
public void testReaderGetRecordIteratorByKeyPrefixes() throws Exception {
writeFileWithSimpleSchema();
try (HoodieAvroHFileReaderImplBase hfileReader = (HoodieAvroHFileReaderImplBase)
- createReader(HadoopFSUtils.getStorageConf(new Configuration()))) {
+ createReader(HoodieTestUtils.getStorage(getFilePath()))) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
@@ -455,8 +453,8 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
verifyHFileReader(
content, hfilePrefix, true, HFILE_COMPARATOR.getClass(), NUM_RECORDS_FIXTURE);
- StorageConfiguration storageConf = HadoopFSUtils.getStorageConf(fs.getConf());
- try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storageConf, content)) {
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storage, content)) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
assertEquals(NUM_RECORDS_FIXTURE, hfileReader.getTotalRecords());
@@ -466,7 +464,7 @@ public abstract class TestHoodieHFileReaderWriterBase extends TestHoodieReaderWr
content = readHFileFromResources(complexHFile);
verifyHFileReader(
content, hfilePrefix, true, HFILE_COMPARATOR.getClass(), NUM_RECORDS_FIXTURE);
- try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storageConf, content)) {
+ try (HoodieAvroHFileReaderImplBase hfileReader = createHFileReader(storage, content)) {
Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchemaWithUDT.avsc");
assertEquals(NUM_RECORDS_FIXTURE, hfileReader.getTotalRecords());
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java
index 314334365b2..0b735ce9f16 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java
@@ -30,6 +30,7 @@ import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.io.storage.HoodieOrcConfig;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
@@ -77,15 +78,15 @@ public class TestHoodieOrcReaderWriter extends TestHoodieReaderWriterBase {
@Override
protected HoodieAvroFileReader createReader(
- StorageConfiguration<?> conf) throws Exception {
- return (HoodieAvroFileReader) HoodieIOFactory.getIOFactory(conf).getReaderFactory(HoodieRecordType.AVRO)
+ HoodieStorage storage) throws Exception {
+ return (HoodieAvroFileReader) HoodieIOFactory.getIOFactory(storage).getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, getFilePath());
}
@Override
- protected void verifyMetadata(StorageConfiguration<?> conf) throws IOException {
+ protected void verifyMetadata(HoodieStorage storage) throws IOException {
Reader orcReader = OrcFile.createReader(
- new Path(getFilePath().toUri()), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)));
+ new Path(getFilePath().toUri()), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)));
assertEquals(4, orcReader.getMetadataKeys().size());
assertTrue(orcReader.getMetadataKeys().contains(HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER));
assertTrue(orcReader.getMetadataKeys().contains(HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER));
@@ -96,9 +97,9 @@ public class TestHoodieOrcReaderWriter extends TestHoodieReaderWriterBase {
}
@Override
- protected void verifySchema(StorageConfiguration<?> conf, String schemaPath) throws IOException {
+ protected void verifySchema(HoodieStorage storage, String schemaPath) throws IOException {
Reader orcReader = OrcFile.createReader(
- new Path(getFilePath().toUri()), OrcFile.readerOptions(conf.unwrapAs(Configuration.class)));
+ new Path(getFilePath().toUri()), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)));
if ("/exampleSchema.avsc".equals(schemaPath)) {
assertEquals("struct<_row_key:string,time:string,number:int>",
orcReader.getSchema().toString());
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieReaderWriterBase.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieReaderWriterBase.java
index 1bd376e4139..f39b0fa66de 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieReaderWriterBase.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieReaderWriterBase.java
@@ -23,19 +23,18 @@ import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieAvroFileWriter;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileWriter;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -75,11 +74,11 @@ public abstract class TestHoodieReaderWriterBase {
Schema avroSchema, boolean populateMetaFields) throws Exception;
protected abstract HoodieAvroFileReader createReader(
- StorageConfiguration<?> conf) throws Exception;
+ HoodieStorage storage) throws Exception;
- protected abstract void verifyMetadata(StorageConfiguration<?> conf) throws IOException;
+ protected abstract void verifyMetadata(HoodieStorage storage) throws IOException;
- protected abstract void verifySchema(StorageConfiguration<?> conf, String schemaPath) throws IOException;
+ protected abstract void verifySchema(HoodieStorage storage, String schemaPath) throws IOException;
@BeforeEach
@AfterEach
@@ -95,10 +94,10 @@ public abstract class TestHoodieReaderWriterBase {
Schema avroSchema = getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
writeFileWithSimpleSchema();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
- verifyMetadata(conf);
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ verifyMetadata(storage);
- try (HoodieAvroFileReader hoodieReader = createReader(conf)) {
+ try (HoodieAvroFileReader hoodieReader = createReader(storage)) {
BloomFilter filter = hoodieReader.readBloomFilter();
for (int i = 0; i < NUM_RECORDS; i++) {
String key = "key" + String.format("%02d", i);
@@ -119,10 +118,10 @@ public abstract class TestHoodieReaderWriterBase {
String schemaPath = "/exampleSchema.avsc";
writeFileWithSimpleSchema();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
- verifyMetadata(conf);
- verifySchema(conf, schemaPath);
- verifySimpleRecords(createReader(conf).getRecordIterator());
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ verifyMetadata(storage);
+ verifySchema(storage, schemaPath);
+ verifySimpleRecords(createReader(storage).getRecordIterator());
}
@Test
@@ -146,10 +145,10 @@ public abstract class TestHoodieReaderWriterBase {
}
writer.close();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
- verifyMetadata(conf);
- verifySchema(conf, schemaPath);
- verifyComplexRecords(createReader(conf).getRecordIterator());
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ verifyMetadata(storage);
+ verifySchema(storage, schemaPath);
+ verifyComplexRecords(createReader(storage).getRecordIterator());
}
@ParameterizedTest
@@ -162,8 +161,8 @@ public abstract class TestHoodieReaderWriterBase {
})
public void testWriteReadWithEvolvedSchema(String evolvedSchemaPath) throws Exception {
writeFileWithSimpleSchema();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
- try (HoodieAvroFileReader hoodieReader = createReader(conf)) {
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ try (HoodieAvroFileReader hoodieReader = createReader(storage)) {
verifyReaderWithSchema(evolvedSchemaPath, hoodieReader);
}
}
@@ -171,9 +170,9 @@ public abstract class TestHoodieReaderWriterBase {
@Test
public void testReaderFilterRowKeys() throws Exception {
writeFileWithSchemaWithMeta();
- StorageConfiguration<?> conf = HadoopFSUtils.getStorageConf(new Configuration());
- verifyMetadata(conf);
- verifyFilterRowKeys(createReader(conf));
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ verifyMetadata(storage);
+ verifyFilterRowKeys(createReader(storage));
}
protected void writeFileWithSimpleSchema() throws Exception {
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
index 80045b9bc63..974db76b3bf 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
@@ -72,7 +72,7 @@ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness {
HoodieLocalEngineContext localEngineContext =
new HoodieLocalEngineContext(metaClient.getStorageConf());
FileSystemBackedTableMetadata fileSystemBackedTableMetadata =
- new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorageConf(), basePath, false);
+ new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorage(), basePath, false);
Assertions.assertEquals(0, fileSystemBackedTableMetadata.getAllPartitionPaths().size());
Assertions.assertEquals(10,
fileSystemBackedTableMetadata.getAllFilesInPartition(new StoragePath(basePath)).size());
@@ -98,7 +98,7 @@ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness {
});
HoodieLocalEngineContext localEngineContext = new HoodieLocalEngineContext(metaClient.getStorageConf());
FileSystemBackedTableMetadata fileSystemBackedTableMetadata =
- new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorageConf(), basePath, true);
+ new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorage(), basePath, true);
Assertions.assertEquals(3, fileSystemBackedTableMetadata.getAllPartitionPaths().size());
Assertions.assertEquals(10, fileSystemBackedTableMetadata.getAllFilesInPartition(new StoragePath(basePath + "/" + DATE_PARTITIONS.get(0))).size());
@@ -130,7 +130,7 @@ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness {
HoodieLocalEngineContext localEngineContext =
new HoodieLocalEngineContext(metaClient.getStorageConf());
FileSystemBackedTableMetadata fileSystemBackedTableMetadata =
- new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorageConf(), basePath, false);
+ new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorage(), basePath, false);
Assertions.assertEquals(3, fileSystemBackedTableMetadata.getAllPartitionPaths().size());
List<String> fullPartitionPaths =
@@ -158,7 +158,7 @@ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness {
HoodieLocalEngineContext localEngineContext =
new HoodieLocalEngineContext(metaClient.getStorageConf());
FileSystemBackedTableMetadata fileSystemBackedTableMetadata =
- new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorageConf(), basePath, false);
+ new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorage(), basePath, false);
Assertions.assertEquals(3, fileSystemBackedTableMetadata.getAllPartitionPaths().size());
Assertions.assertEquals(10, fileSystemBackedTableMetadata.getAllFilesInPartition(
new StoragePath(basePath + "/" + ONE_LEVEL_PARTITIONS.get(0))).size());
@@ -188,7 +188,7 @@ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness {
HoodieLocalEngineContext localEngineContext =
new HoodieLocalEngineContext(metaClient.getStorageConf());
FileSystemBackedTableMetadata fileSystemBackedTableMetadata =
- new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorageConf(), basePath, false);
+ new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorage(), basePath, false);
Assertions.assertEquals(3, fileSystemBackedTableMetadata.getAllPartitionPaths().size());
Assertions.assertEquals(10, fileSystemBackedTableMetadata.getAllFilesInPartition(
new StoragePath(basePath + "/" + MULTI_LEVEL_PARTITIONS.get(0))).size());
@@ -217,7 +217,7 @@ public class TestFileSystemBackedTableMetadata extends HoodieCommonTestHarness {
HoodieLocalEngineContext localEngineContext =
new HoodieLocalEngineContext(metaClient.getStorageConf());
FileSystemBackedTableMetadata fileSystemBackedTableMetadata =
- new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorageConf(), basePath, false);
+ new FileSystemBackedTableMetadata(localEngineContext, metaClient.getTableConfig(), metaClient.getStorage(), basePath, false);
Assertions.assertEquals(3, fileSystemBackedTableMetadata.getAllPartitionPaths().size());
Assertions.assertEquals(0, fileSystemBackedTableMetadata.getAllFilesInPartition(
new StoragePath(basePath + "/" + MULTI_LEVEL_PARTITIONS.get(0))).size());
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
index c66ec4265a4..065ce5e3171 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
@@ -141,7 +141,7 @@ public class TestHoodieTableMetadataUtil extends HoodieCommonTestHarness {
HoodieFileWriter writer = HoodieFileWriterFactory.getFileWriter(
instant,
path,
- metaClient.getStorageConf(),
+ metaClient.getStorage(),
metaClient.getTableConfig(),
HoodieTestDataGenerator.AVRO_SCHEMA_WITH_METADATA_FIELDS,
engineContext.getTaskContextSupplier(),
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileRecordReader.java
index 1d057901908..3414f5fdea8 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileRecordReader.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileRecordReader.java
@@ -29,6 +29,7 @@ import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
@@ -59,7 +60,7 @@ public class HoodieHFileRecordReader implements RecordReader<NullWritable, Array
StoragePath path = convertToStoragePath(fileSplit.getPath());
StorageConfiguration<?> storageConf = HadoopFSUtils.getStorageConf(conf);
HoodieConfig hoodieConfig = getReaderConfigs(storageConf);
- reader = HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
+ reader = HoodieIOFactory.getIOFactory(new HoodieHadoopStorage(path, storageConf)).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, path, HoodieFileFormat.HFILE, Option.empty());
schema = reader.getSchema();
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
index 51d8a9f3af4..3b465abd724 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
@@ -30,9 +30,9 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.hadoop.utils.HoodieHiveUtils;
import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -133,8 +133,7 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial
Path folder = null;
try {
if (storage == null) {
- storage =
- HoodieStorageUtils.getStorage(convertToStoragePath(path), conf);
+ storage = new HoodieHadoopStorage(convertToStoragePath(path), conf);
}
// Assumes path is a file
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java
index 6945b241e0a..fe88855d458 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java
@@ -48,6 +48,7 @@ import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -524,10 +525,10 @@ public class HoodieInputFormatUtils {
List<HoodieCommitMetadata> metadataList) {
// TODO: Use HoodieMetaTable to extract affected file directly.
HashMap<String, StoragePathInfo> fullPathToInfoMap = new HashMap<>();
+ HoodieStorage storage = new HoodieHadoopStorage(basePath, HadoopFSUtils.getStorageConf(hadoopConf));
// Iterate through the given commits.
for (HoodieCommitMetadata metadata : metadataList) {
- fullPathToInfoMap.putAll(metadata.getFullPathToInfo(
- HadoopFSUtils.getStorageConf(hadoopConf), basePath.toString()));
+ fullPathToInfoMap.putAll(metadata.getFullPathToInfo(storage, basePath.toString()));
}
return new ArrayList<>(fullPathToInfoMap.values());
}
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
index a612ab4616c..9ed34b8619d 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.avro.JsonProperties;
import org.apache.avro.LogicalType;
@@ -309,7 +310,8 @@ public class HoodieRealtimeRecordReaderUtils {
public static HoodieFileReader getBaseFileReader(Path path, JobConf conf) throws IOException {
StorageConfiguration<?> storageConf = HadoopFSUtils.getStorageConf(conf);
HoodieConfig hoodieConfig = getReaderConfigs(storageConf);
- return HoodieIOFactory.getIOFactory(storageConf).getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
+ return HoodieIOFactory.getIOFactory(new HoodieHadoopStorage(path, conf))
+ .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, convertToStoragePath(path));
}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
index 6c3286a47bf..3243f0a0fbb 100644
--- a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
@@ -97,7 +97,7 @@ public class DFSHoodieDatasetInputReader extends DFSDeltaInputReader {
// Using FSUtils.getFS here instead of metaClient.getFS() since we don't want to count these listStatus
// calls in metrics as they are not part of normal HUDI operation.
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
- List<String> partitionPaths = FSUtils.getAllPartitionPaths(engineContext, metaClient.getBasePath(),
+ List<String> partitionPaths = FSUtils.getAllPartitionPaths(engineContext, metaClient.getStorage(), metaClient.getBasePath(),
HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false);
// Sort partition so we can pick last N partitions by default
Collections.sort(partitionPaths);
@@ -274,7 +274,7 @@ public class DFSHoodieDatasetInputReader extends DFSDeltaInputReader {
if (fileSlice.getBaseFile().isPresent()) {
// Read the base files using the latest writer schema.
Schema schema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(schemaStr));
- HoodieAvroFileReader reader = TypeUtils.unsafeCast(HoodieIOFactory.getIOFactory(metaClient.getStorageConf())
+ HoodieAvroFileReader reader = TypeUtils.unsafeCast(HoodieIOFactory.getIOFactory(metaClient.getStorage())
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(
DEFAULT_HUDI_CONFIG_FOR_READER,
diff --git a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
index a92a2938bfc..fc5ae52de54 100644
--- a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
+++ b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
@@ -47,6 +47,21 @@ import java.util.UUID;
public abstract class HoodieStorage implements Closeable {
public static final Logger LOG = LoggerFactory.getLogger(HoodieStorage.class);
+ protected final StorageConfiguration<?> storageConf;
+
+ public HoodieStorage(StorageConfiguration<?> storageConf) {
+ this.storageConf = storageConf;
+ }
+
+ /**
+ * @param path path to instantiate the storage.
+ * @param storageConf new storage configuration.
+ * @return new {@link HoodieStorage} instance with the configuration.
+ */
+ @PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
+ public abstract HoodieStorage newInstance(StoragePath path,
+ StorageConfiguration<?> storageConf);
+
/**
* @return the scheme of the storage.
*/
@@ -257,16 +272,18 @@ public abstract class HoodieStorage implements Closeable {
public abstract Object getFileSystem();
/**
- * @return the storage configuration.
+ * @return the raw storage.
*/
@PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
- public abstract StorageConfiguration<?> getConf();
+ public abstract HoodieStorage getRawStorage();
/**
- * @return the raw storage.
+ * @return the storage configuration.
*/
@PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
- public abstract HoodieStorage getRawStorage();
+ public final StorageConfiguration<?> getConf() {
+ return storageConf;
+ }
/**
* Creates a new file with overwrite set to false. This ensures files are created
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java
index ad17abd294a..d237c2d6560 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java
@@ -48,7 +48,9 @@ public class DatasetBulkInsertOverwriteTableCommitActionExecutor extends Dataset
@Override
protected Map<String, List<String>> getPartitionToReplacedFileIds(HoodieData<WriteStatus> writeStatuses) {
HoodieEngineContext context = writeClient.getEngineContext();
- List<String> partitionPaths = FSUtils.getAllPartitionPaths(context, writeConfig.getMetadataConfig(),
+ List<String> partitionPaths = FSUtils.getAllPartitionPaths(context,
+ table.getStorage(),
+ writeConfig.getMetadataConfig(),
table.getMetaClient().getBasePathV2().toString());
if (partitionPaths == null || partitionPaths.isEmpty()) {
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala
index f5a5b14eaad..03a8a6c677e 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala
@@ -60,7 +60,7 @@ class ColumnStatsIndexSupport(spark: SparkSession,
@transient private lazy val engineCtx = new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext))
@transient private lazy val metadataTable: HoodieTableMetadata =
- HoodieTableMetadata.create(engineCtx, metadataConfig, metaClient.getBasePathV2.toString)
+ HoodieTableMetadata.create(engineCtx, metaClient.getStorage, metadataConfig, metaClient.getBasePathV2.toString)
@transient private lazy val cachedColumnStatsIndexViews: ParHashMap[Seq[String], DataFrame] = ParHashMap()
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
index d8f6f48878a..fde7884daa2 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
@@ -40,18 +40,20 @@ import org.apache.hudi.config.HoodieBootstrapConfig.DATA_QUERIES_ONLY
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.exception.HoodieException
import org.apache.hudi.hadoop.fs.HadoopFSUtils
+import org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
import org.apache.hudi.io.storage.HoodieSparkIOFactory
import org.apache.hudi.metadata.HoodieTableMetadata
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hudi.storage.{StoragePath, StoragePathInfo}
+
import org.apache.avro.Schema
import org.apache.avro.generic.GenericRecord
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileStatus, Path}
import org.apache.hadoop.mapred.JobConf
-import org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath
import org.apache.spark.execution.datasources.HoodieInMemoryFileIndex
import org.apache.spark.internal.Logging
import org.apache.spark.rdd.RDD
@@ -756,7 +758,8 @@ object HoodieBaseRelation extends SparkAdapterSupport {
val hoodieConfig = new HoodieConfig()
hoodieConfig.setValue(USE_NATIVE_HFILE_READER,
options.getOrElse(USE_NATIVE_HFILE_READER.key(), USE_NATIVE_HFILE_READER.defaultValue().toString))
- val reader = new HoodieSparkIOFactory(storageConf).getReaderFactory(HoodieRecordType.AVRO)
+ val reader = new HoodieSparkIOFactory(new HoodieHadoopStorage(filePath, storageConf))
+ .getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, filePath, HFILE)
val requiredRowSchema = requiredDataSchema.structTypeSchema
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
index 41657377753..5b9b57cf10c 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
@@ -55,6 +55,7 @@ import org.apache.hudi.keygen.constant.KeyGeneratorType
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory
import org.apache.hudi.keygen.{BaseKeyGenerator, TimestampBasedAvroKeyGenerator, TimestampBasedKeyGenerator}
import org.apache.hudi.metrics.Metrics
+import org.apache.hudi.storage.HoodieStorage
import org.apache.hudi.sync.common.HoodieSyncConfig
import org.apache.hudi.sync.common.util.SyncUtilHelpers
import org.apache.hudi.sync.common.util.SyncUtilHelpers.getHoodieMetaSyncException
@@ -387,20 +388,20 @@ class HoodieSparkSqlWriterInternal {
}
val keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(new TypedProperties(hoodieConfig.getProps))
+ val tableMetaClient = HoodieTableMetaClient.builder
+ .setConf(HadoopFSUtils.getStorageConfWithCopy(sparkContext.hadoopConfiguration))
+ .setBasePath(basePath.toString).build()
// Get list of partitions to delete
val partitionsToDelete = if (parameters.contains(DataSourceWriteOptions.PARTITIONS_TO_DELETE.key())) {
val partitionColsToDelete = parameters(DataSourceWriteOptions.PARTITIONS_TO_DELETE.key()).split(",")
java.util.Arrays.asList(resolvePartitionWildcards(java.util.Arrays.asList(partitionColsToDelete: _*).asScala.toList, jsc,
- hoodieConfig, basePath.toString): _*)
+ tableMetaClient.getStorage, hoodieConfig, basePath.toString): _*)
} else {
val genericRecords = HoodieSparkUtils.createRdd(df, avroRecordName, avroRecordNamespace)
genericRecords.map(gr => keyGenerator.getKey(gr).getPartitionPath).toJavaRDD().distinct().collect()
}
- // Create a HoodieWriteClient & issue the delete.
- val tableMetaClient = HoodieTableMetaClient.builder
- .setConf(HadoopFSUtils.getStorageConfWithCopy(sparkContext.hadoopConfiguration))
- .setBasePath(basePath.toString).build()
+ // Issue the delete.
val schemaStr = new TableSchemaResolver(tableMetaClient).getTableAvroSchema.toString
val client = hoodieWriteClient.getOrElse(DataSourceUtils.createHoodieClient(jsc,
schemaStr, path, tblName,
@@ -684,19 +685,20 @@ class HoodieSparkSqlWriterInternal {
*
* @param partitions list of partitions that may contain wildcards
* @param jsc instance of java spark context
+ * @param storage [[HoodieStorage]] instance
* @param cfg hoodie config
* @param basePath base path
* @return Pair of(boolean, table schema), where first entry will be true only if schema conversion is required.
*/
- private def resolvePartitionWildcards(partitions: List[String], jsc: JavaSparkContext, cfg: HoodieConfig, basePath: String): List[String] = {
+ private def resolvePartitionWildcards(partitions: List[String], jsc: JavaSparkContext,
+ storage: HoodieStorage, cfg: HoodieConfig, basePath: String): List[String] = {
//find out if any of the input partitions have wildcards
//note:spark-sql may url-encode special characters (* -> %2A)
var (wildcardPartitions, fullPartitions) = partitions.partition(partition => partition.matches(".*(\\*|%2A).*"))
if (wildcardPartitions.nonEmpty) {
//get list of all partitions
- val allPartitions = FSUtils.getAllPartitionPaths(new HoodieSparkEngineContext(jsc): HoodieEngineContext,
- HoodieMetadataConfig.newBuilder().fromProperties(cfg.getProps).build(), basePath)
+ val allPartitions = FSUtils.getAllPartitionPaths(new HoodieSparkEngineContext(jsc): HoodieEngineContext, storage, HoodieMetadataConfig.newBuilder().fromProperties(cfg.getProps).build(), basePath)
//go through list of partitions with wildcards and add all partitions that match to val fullPartitions
wildcardPartitions.foreach(partition => {
//turn wildcard into regex
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala
index de32136e910..1cf8bc5b655 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala
@@ -353,7 +353,7 @@ object LogFileIterator extends SparkAdapterSupport {
.enable(true).build()
val dataTableBasePath = getDataTableBasePathFromMetadataTable(tablePath)
val metadataTable = new HoodieBackedTableMetadata(
- new HoodieLocalEngineContext(HadoopFSUtils.getStorageConf(hadoopConf)), metadataConfig,
+ new HoodieLocalEngineContext(HadoopFSUtils.getStorageConf(hadoopConf)), storage, metadataConfig,
dataTableBasePath)
// We have to force full-scan for the MT log record reader, to make sure
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala
index 1b72df1c97d..257a26db7e0 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala
@@ -30,7 +30,7 @@ import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, In, Literal}
-import scala.collection.{mutable, JavaConverters}
+import scala.collection.{JavaConverters, mutable}
class RecordLevelIndexSupport(spark: SparkSession,
metadataConfig: HoodieMetadataConfig,
@@ -38,7 +38,7 @@ class RecordLevelIndexSupport(spark: SparkSession,
@transient private lazy val engineCtx = new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext))
@transient private lazy val metadataTable: HoodieTableMetadata =
- HoodieTableMetadata.create(engineCtx, metadataConfig, metaClient.getBasePathV2.toString)
+ HoodieTableMetadata.create(engineCtx, metaClient.getStorage, metadataConfig, metaClient.getBasePathV2.toString)
/**
* Returns the list of candidate files which store the provided record keys based on Metadata Table Record Index.
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
index 961759c73b7..505b4b55bea 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
@@ -173,7 +173,7 @@ class HoodieCatalogTable(val spark: SparkSession, var table: CatalogTable) exten
def getPartitionPaths: Seq[String] = {
val droppedPartitions = TimelineUtils.getDroppedPartitions(metaClient, org.apache.hudi.common.util.Option.empty(), org.apache.hudi.common.util.Option.empty())
- getAllPartitionPaths(spark, table)
+ getAllPartitionPaths(spark, table, metaClient.getStorage)
.filter(!droppedPartitions.contains(_))
}
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
index b95f760d8b4..645e528d730 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
@@ -17,29 +17,28 @@
package org.apache.spark.sql.hudi
-import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, SparkAdapterSupport}
import org.apache.hudi.client.common.HoodieSparkEngineContext
-import org.apache.hudi.common.config.{DFSPropertiesConfiguration, HoodieMetadataConfig, TypedProperties}
+import org.apache.hudi.common.config.{HoodieMetadataConfig, TypedProperties}
+import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.model.HoodieRecord
-import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
-import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieInstantTimeGenerator, HoodieTimeline}
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline.parseDateFromInstantTime
+import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieInstantTimeGenerator, HoodieTimeline}
+import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
import org.apache.hudi.common.util.PartitionPathEncodeUtils
import org.apache.hudi.exception.HoodieException
+import org.apache.hudi.storage.{HoodieStorage, StoragePathInfo}
import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, SparkAdapterSupport}
-import org.apache.hudi.common.fs.FSUtils
-import org.apache.hudi.storage.StoragePathInfo
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark.api.java.JavaSparkContext
-import org.apache.spark.sql.{AnalysisException, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.Resolver
import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HoodieCatalogTable}
import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression, Literal}
import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
import org.apache.spark.sql.types._
+import org.apache.spark.sql.{AnalysisException, SparkSession}
import java.net.URI
import java.text.SimpleDateFormat
@@ -66,24 +65,25 @@ object HoodieSqlCommonUtils extends SparkAdapterSupport {
avroSchema.map(AvroConversionUtils.convertAvroSchemaToStructType)
}
- def getAllPartitionPaths(spark: SparkSession, table: CatalogTable): Seq[String] = {
+ def getAllPartitionPaths(spark: SparkSession, table: CatalogTable, storage: HoodieStorage): Seq[String] = {
val sparkEngine = new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext))
val metadataConfig = {
val properties = TypedProperties.fromMap((spark.sessionState.conf.getAllConfs ++ table.storage.properties ++ table.properties).asJava)
HoodieMetadataConfig.newBuilder.fromProperties(properties).build()
}
- FSUtils.getAllPartitionPaths(sparkEngine, metadataConfig, getTableLocation(table, spark)).asScala.toSeq
+ FSUtils.getAllPartitionPaths(sparkEngine, storage, metadataConfig, getTableLocation(table, spark)).asScala.toSeq
}
def getFilesInPartitions(spark: SparkSession,
table: CatalogTable,
+ storage: HoodieStorage,
partitionPaths: Seq[String]): Map[String, Seq[StoragePathInfo]] = {
val sparkEngine = new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext))
val metadataConfig = {
val properties = TypedProperties.fromMap((spark.sessionState.conf.getAllConfs ++ table.storage.properties ++ table.properties).asJava)
HoodieMetadataConfig.newBuilder.fromProperties(properties).build()
}
- FSUtils.getFilesInPartitions(sparkEngine, metadataConfig, getTableLocation(table, spark),
+ FSUtils.getFilesInPartitions(sparkEngine, storage, metadataConfig, getTableLocation(table, spark),
partitionPaths.toArray).asScala
.map(e => (e._1, e._2.asScala.toSeq))
.toMap
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala
index 8dc61c32531..53d131d78a5 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala
@@ -17,17 +17,21 @@
package org.apache.spark.sql.hudi.command
+import org.apache.hudi.hadoop.fs.HadoopFSUtils
+import org.apache.hudi.storage.HoodieStorageUtils
+
import org.apache.hadoop.fs.Path
-import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.execution.command.PartitionStatistics
import org.apache.spark.sql.hudi.HoodieSqlCommonUtils
+import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
import org.apache.spark.util.ThreadUtils
import java.util.concurrent.TimeUnit.MILLISECONDS
+import scala.language.postfixOps
import scala.util.control.NonFatal
/**
@@ -81,9 +85,11 @@ case class RepairHoodieTableCommand(tableName: TableIdentifier,
} else 0
val addedAmount = if (enableAddPartitions) {
val total = partitionSpecsAndLocs.length
- val partitionStats = if (spark.sqlContext.conf.gatherFastStats) {
- HoodieSqlCommonUtils.getFilesInPartitions(spark, table, partitionSpecsAndLocs
- .map(_._2.toString))
+ val partitionList = partitionSpecsAndLocs.map(_._2.toString)
+ val partitionStats = if (spark.sqlContext.conf.gatherFastStats && total > 0) {
+ HoodieSqlCommonUtils.getFilesInPartitions(spark, table,
+ HoodieStorageUtils.getStorage(partitionList.head, HadoopFSUtils.getStorageConf(spark.sessionState.newHadoopConf())),
+ partitionList)
.mapValues(statuses => PartitionStatistics(statuses.length, statuses.map(_.getLength).sum))
} else {
Map.empty[String, PartitionStatistics]
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala
index f0781895c4d..9b3405432a5 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala
@@ -26,6 +26,7 @@ import org.apache.hudi.common.table.timeline.TimelineUtils.{HollowCommitHandling
import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
import org.apache.hudi.common.util.TablePathUtils
import org.apache.hudi.hadoop.fs.HadoopFSUtils
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hudi.storage.{HoodieStorageUtils, StoragePath}
import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, IncrementalRelation, MergeOnReadIncrementalRelation, SparkAdapterSupport}
@@ -58,7 +59,7 @@ class HoodieStreamSource(
private lazy val tablePath: StoragePath = {
val path = new StoragePath(parameters.getOrElse("path", "Missing 'path' option"))
- val fs = HoodieStorageUtils.getStorage(path, storageConf)
+ val fs = new HoodieHadoopStorage(path, storageConf)
TablePathUtils.getTablePath(fs, path).get()
}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala
index c2a717e2764..e534a13d766 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala
@@ -48,7 +48,7 @@ object SparkHelpers {
sourceFile: StoragePath,
destinationFile: StoragePath,
keysToSkip: Set[String]) {
- val sourceRecords = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET).readAvroRecords(conf, sourceFile).asScala
+ val sourceRecords = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET).readAvroRecords(storage, sourceFile).asScala
val schema: Schema = sourceRecords.head.getSchema
val filter: BloomFilter = BloomFilterFactory.createBloomFilter(
BLOOM_FILTER_NUM_ENTRIES_VALUE.defaultValue.toInt, BLOOM_FILTER_FPP_VALUE.defaultValue.toDouble,
@@ -134,13 +134,13 @@ class SparkHelper(sqlContext: SQLContext, fs: FileSystem) {
* Checks that all the keys in the file, have been added to the bloom filter
* in the footer
*
- * @param conf
- * @param sqlContext
- * @param file
- * @return
+ * @param storage [[HoodieStorage]] instance.
+ * @param sqlContext Spark SQL context.
+ * @param file File path.
+ * @return <pre>true</pre> if all keys are added to the bloom filter; <pre>false</pre> otherwise.
*/
- def fileKeysAgainstBF(conf: StorageConfiguration[_], sqlContext: SQLContext, file: String): Boolean = {
- val bf = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET).readBloomFilterFromMetadata(conf, new StoragePath(file))
+ def fileKeysAgainstBF(storage: HoodieStorage, sqlContext: SQLContext, file: String): Boolean = {
+ val bf = FileFormatUtils.getInstance(HoodieFileFormat.PARQUET).readBloomFilterFromMetadata(storage, new StoragePath(file))
val foundCount = sqlContext.parquetFile(file)
.select(s"`${HoodieRecord.RECORD_KEY_METADATA_FIELD}`")
.collect().count(r => !bf.mightContain(r.getString(0)))
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
index 292f6d5fdee..dc3f114b32b 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
@@ -33,6 +33,7 @@ import java.io.IOException
import java.util
import java.util.Properties
import java.util.function.{Consumer, Supplier}
+
import scala.collection.JavaConverters._
class RepairMigratePartitionMetaProcedure extends BaseProcedure with ProcedureBuilder with Logging {
@@ -62,7 +63,7 @@ class RepairMigratePartitionMetaProcedure extends BaseProcedure with ProcedureBu
val metaClient = createMetaClient(jsc, tablePath)
val engineContext: HoodieLocalEngineContext = new HoodieLocalEngineContext(metaClient.getStorageConf)
- val partitionPaths: util.List[String] = FSUtils.getAllPartitionPaths(engineContext, tablePath, false, false)
+ val partitionPaths: util.List[String] = FSUtils.getAllPartitionPaths(engineContext, metaClient.getStorage, tablePath, false, false)
val basePath: StoragePath = new StoragePath(tablePath)
val rows = new util.ArrayList[Row](partitionPaths.size)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala
index dacfdef6739..0125d4da130 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.client.common.HoodieSparkEngineContext
import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.hadoop.fs.HadoopFSUtils
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.fs.Path
import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS
@@ -47,9 +48,10 @@ class ShowInvalidParquetProcedure extends BaseProcedure with ProcedureBuilder {
super.checkArgs(PARAMETERS, args)
val srcPath = getArgValueOrDefault(args, PARAMETERS(0)).get.asInstanceOf[String]
- val partitionPaths: java.util.List[String] = FSUtils.getAllPartitionPaths(new HoodieSparkEngineContext(jsc), srcPath, false, false)
- val javaRdd: JavaRDD[String] = jsc.parallelize(partitionPaths, partitionPaths.size())
val storageConf = HadoopFSUtils.getStorageConfWithCopy(jsc.hadoopConfiguration())
+ val storage = new HoodieHadoopStorage(srcPath, storageConf)
+ val partitionPaths: java.util.List[String] = FSUtils.getAllPartitionPaths(new HoodieSparkEngineContext(jsc), storage, srcPath, false, false)
+ val javaRdd: JavaRDD[String] = jsc.parallelize(partitionPaths, partitionPaths.size())
javaRdd.rdd.map(part => {
val fs = HadoopFSUtils.getFs(new Path(srcPath), storageConf.unwrap())
HadoopFSUtils.getAllDataFilesInPartition(fs, HadoopFSUtils.constructAbsolutePathInHadoopPath(srcPath, part))
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala
index 719e24ae7d9..83d22064975 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala
@@ -21,7 +21,6 @@ import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.engine.HoodieLocalEngineContext
import org.apache.hudi.common.util.{HoodieTimer, StringUtils}
import org.apache.hudi.exception.HoodieException
-import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.metadata.HoodieBackedTableMetadata
import org.apache.hudi.storage.{StoragePath, StoragePathInfo}
@@ -57,7 +56,8 @@ class ShowMetadataTableFilesProcedure() extends BaseProcedure with ProcedureBuil
val basePath = getBasePath(table)
val metaClient = createMetaClient(jsc, basePath)
val config = HoodieMetadataConfig.newBuilder.enable(true).build
- val metaReader = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getStorageConf), config, basePath)
+ val metaReader = new HoodieBackedTableMetadata(
+ new HoodieLocalEngineContext(metaClient.getStorageConf), metaClient.getStorage, config, basePath)
if (!metaReader.enabled){
throw new HoodieException(s"Metadata Table not enabled/initialized.")
}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala
index 9a7867d6648..9f0e23a86fe 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala
@@ -21,7 +21,10 @@ import org.apache.hudi.client.common.HoodieSparkEngineContext
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.util.HoodieTimer
import org.apache.hudi.exception.HoodieException
+import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.metadata.HoodieBackedTableMetadata
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
+
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -29,6 +32,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
import java.util.Collections
import java.util.function.Supplier
+
import scala.collection.JavaConverters.asScalaIteratorConverter
class ShowMetadataTablePartitionsProcedure() extends BaseProcedure with ProcedureBuilder with Logging {
@@ -50,8 +54,9 @@ class ShowMetadataTablePartitionsProcedure() extends BaseProcedure with Procedur
val table = getArgValueOrDefault(args, PARAMETERS(0))
val basePath = getBasePath(table)
+ val storage = new HoodieHadoopStorage(basePath, spark.sessionState.newHadoopConf())
val config = HoodieMetadataConfig.newBuilder.enable(true).build
- val metadata = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), config, basePath)
+ val metadata = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), storage, config, basePath)
if (!metadata.enabled){
throw new HoodieException(s"Metadata Table not enabled/initialized.")
}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala
index f67c6ac5406..37e6c92fa23 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
import java.util.function.Supplier
+
import scala.collection.JavaConverters._
class ShowMetadataTableStatsProcedure() extends BaseProcedure with ProcedureBuilder {
@@ -50,7 +51,8 @@ class ShowMetadataTableStatsProcedure() extends BaseProcedure with ProcedureBuil
val basePath = getBasePath(table)
val metaClient = createMetaClient(jsc, basePath)
val config = HoodieMetadataConfig.newBuilder.enable(true).build
- val metadata = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getStorageConf), config, basePath)
+ val metadata = new HoodieBackedTableMetadata(
+ new HoodieLocalEngineContext(metaClient.getStorageConf), metaClient.getStorage, config, basePath)
val stats = metadata.stats
val rows = new util.ArrayList[Row]
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala
index 61157bb9253..c1d79a768fa 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala
@@ -21,7 +21,6 @@ import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.engine.HoodieLocalEngineContext
import org.apache.hudi.common.util.HoodieTimer
import org.apache.hudi.exception.HoodieException
-import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.metadata.HoodieBackedTableMetadata
import org.apache.hudi.storage.{StoragePath, StoragePathInfo}
@@ -64,7 +63,7 @@ class ValidateMetadataTableFilesProcedure() extends BaseProcedure with Procedure
val metaClient = createMetaClient(jsc, basePath)
val config = HoodieMetadataConfig.newBuilder.enable(true).build
val metadataReader = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getStorageConf),
- config, basePath)
+ metaClient.getStorage, config, basePath)
if (!metadataReader.enabled) {
throw new HoodieException(s"Metadata Table not enabled/initialized.")
@@ -72,7 +71,7 @@ class ValidateMetadataTableFilesProcedure() extends BaseProcedure with Procedure
val fsConfig = HoodieMetadataConfig.newBuilder.enable(false).build
val fsMetaReader = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getStorageConf),
- fsConfig, basePath)
+ metaClient.getStorage, fsConfig, basePath)
val timer = HoodieTimer.start
val metadataPartitions = metadataReader.getAllPartitionPaths
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/ColumnStatsIndexHelper.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/ColumnStatsIndexHelper.java
index 8ff46be7621..6653c9cf969 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/ColumnStatsIndexHelper.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/ColumnStatsIndexHelper.java
@@ -24,8 +24,10 @@ import org.apache.hudi.common.util.ParquetUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.JavaScalaConverters;
import org.apache.spark.SparkContext;
@@ -177,13 +179,15 @@ public class ColumnStatsIndexHelper {
ParquetUtils utils = (ParquetUtils) FileFormatUtils.getInstance(HoodieFileFormat.PARQUET);
Iterable<String> iterable = () -> paths;
return StreamSupport.stream(iterable.spliterator(), false)
- .flatMap(path ->
- utils.readColumnStatsFromMetadata(
- storageConf,
- new StoragePath(path),
- columnNames
- )
- .stream()
+ .flatMap(path -> {
+ HoodieStorage storage = new HoodieHadoopStorage(path, storageConf);
+ return utils.readColumnStatsFromMetadata(
+ storage,
+ new StoragePath(path),
+ columnNames
+ )
+ .stream();
+ }
)
.iterator();
})
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java
index feec6c78ab2..ea866123cca 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java
@@ -178,7 +178,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
metaClient, (FileSystem) metaClient.getStorage().getFileSystem(),
srcPath, context).stream().findAny().map(p -> p.getValue().stream().findAny())
.orElse(null).get().getPath()).toString();
- HoodieAvroParquetReader parquetReader = new HoodieAvroParquetReader(metaClient.getStorageConf(), new StoragePath(filePath));
+ HoodieAvroParquetReader parquetReader = new HoodieAvroParquetReader(metaClient.getStorage(), new StoragePath(filePath));
return parquetReader.getSchema();
}
@@ -402,7 +402,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
reloadInputFormats();
List<GenericRecord> records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, roJobConf, false, schema, TRIP_HIVE_COLUMN_TYPES, false, new ArrayList<>());
assertEquals(totalRecords, records.size());
@@ -420,7 +420,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
seenKeys = new HashSet<>();
records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, rtJobConf, true, schema, TRIP_HIVE_COLUMN_TYPES, false, new ArrayList<>());
assertEquals(totalRecords, records.size());
@@ -436,7 +436,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
reloadInputFormats();
records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, roJobConf, false, schema, TRIP_HIVE_COLUMN_TYPES,
true, HoodieRecord.HOODIE_META_COLUMNS);
@@ -453,7 +453,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
seenKeys = new HashSet<>();
records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, rtJobConf, true, schema, TRIP_HIVE_COLUMN_TYPES, true,
HoodieRecord.HOODIE_META_COLUMNS);
@@ -468,7 +468,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
reloadInputFormats();
records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, roJobConf, false, schema, TRIP_HIVE_COLUMN_TYPES, true,
Arrays.asList("_row_key"));
@@ -485,7 +485,7 @@ public class TestBootstrap extends HoodieSparkClientTestBase {
seenKeys = new HashSet<>();
records = HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(
HadoopFSUtils.getStorageConf(jsc.hadoopConfiguration()),
- FSUtils.getAllPartitionPaths(context, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
+ FSUtils.getAllPartitionPaths(context, storage, basePath, HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS, false).stream()
.map(f -> basePath + "/" + f).collect(Collectors.toList()),
basePath, rtJobConf, true, schema, TRIP_HIVE_COLUMN_TYPES, true,
Arrays.asList("_row_key"));
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java
index d5815928ceb..834d8416f42 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java
@@ -191,12 +191,12 @@ public class TestSparkConsistentBucketClustering extends HoodieSparkClientTestHa
StoragePath metadataPath =
FSUtils.constructAbsolutePath(table.getMetaClient().getHashingMetadataPath(), p);
try {
- table.getMetaClient().getStorage().listDirectEntries(metadataPath).forEach(fl -> {
+ table.getStorage().listDirectEntries(metadataPath).forEach(fl -> {
if (fl.getPath().getName()
.contains(HoodieConsistentHashingMetadata.HASHING_METADATA_COMMIT_FILE_SUFFIX)) {
try {
// delete commit marker to test recovery job
- table.getMetaClient().getStorage().deleteDirectory(fl.getPath());
+ table.getStorage().deleteDirectory(fl.getPath());
} catch (IOException e) {
throw new RuntimeException(e);
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
index 95f151336c7..06829365cad 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
@@ -111,7 +111,7 @@ public class TestHoodieInternalRowParquetWriter extends HoodieSparkClientTestHar
String minKey = recordKeys.stream().min(Comparator.naturalOrder()).get();
String maxKey = recordKeys.stream().max(Comparator.naturalOrder()).get();
- FileMetaData parquetMetadata = ParquetUtils.readMetadata(storageConf, filePath).getFileMetaData();
+ FileMetaData parquetMetadata = ParquetUtils.readMetadata(storage, filePath).getFileMetaData();
Map<String, String> extraMetadata = parquetMetadata.getKeyValueMetaData();
@@ -120,7 +120,7 @@ public class TestHoodieInternalRowParquetWriter extends HoodieSparkClientTestHar
assertEquals(extraMetadata.get(HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE), BloomFilterTypeCode.DYNAMIC_V0.name());
// Step 3: Make sure Bloom Filter contains all the record keys
- BloomFilter bloomFilter = new ParquetUtils().readBloomFilterFromMetadata(storageConf, filePath);
+ BloomFilter bloomFilter = new ParquetUtils().readBloomFilterFromMetadata(storage, filePath);
recordKeys.forEach(recordKey -> {
assertTrue(bloomFilter.mightContain(recordKey));
});
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala
index 75af17fe48d..57e172bca5b 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala
@@ -602,6 +602,7 @@ class TestHoodieFileIndex extends HoodieSparkClientTestBase with ScalaAssertionS
// Test getting partition paths in a subset of directories
val metadata = HoodieTableMetadata.create(context,
+ metaClient.getStorage,
HoodieMetadataConfig.newBuilder().enable(enableMetadataTable).build(),
metaClient.getBasePathV2.toString)
assertEquals(
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
index bc8e8da1b81..e297bdae191 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
@@ -181,7 +181,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
assertTrue(snapshot0.filter("_hoodie_partition_path = '" + HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH + "'").count() > 0)
assertTrue(snapshot0.filter("_hoodie_partition_path = '" + HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH + "'").count() > 0)
assertTrue(snapshot0.filter("_hoodie_partition_path = '" + HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH + "'").count() > 0)
- val storage = HoodieStorageUtils.getStorage(new StoragePath(basePath), HoodieTestUtils.getDefaultStorageConf)
+ val storage = HoodieTestUtils.getStorage(new StoragePath(basePath))
assertTrue(storage.exists(new StoragePath(basePath + "/" + HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)))
assertTrue(storage.exists(new StoragePath(basePath + "/" + HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH)))
assertTrue(storage.exists(new StoragePath(basePath + "/" + HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH)))
@@ -206,7 +206,8 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
snapshot1.cache()
assertEquals(300, snapshot1.count())
- var partitionPaths = FSUtils.getAllPartitionPaths(new HoodieSparkEngineContext(jsc), HoodieMetadataConfig.newBuilder().build(), basePath)
+ var partitionPaths = FSUtils.getAllPartitionPaths(
+ new HoodieSparkEngineContext(jsc), storage, HoodieMetadataConfig.newBuilder().build(), basePath)
assertTrue(partitionPaths.contains("100/rider-123"))
assertTrue(partitionPaths.contains("200/rider-456"))
@@ -229,7 +230,8 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.mode(SaveMode.Overwrite)
.save(basePath)
- partitionPaths = FSUtils.getAllPartitionPaths(new HoodieSparkEngineContext(jsc), HoodieMetadataConfig.newBuilder().build(), basePath)
+ partitionPaths = FSUtils.getAllPartitionPaths(
+ new HoodieSparkEngineContext(jsc), storage, HoodieMetadataConfig.newBuilder().build(), basePath)
assertEquals(partitionPaths.size(), 1)
assertEquals(partitionPaths.get(0), "")
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala
index 32a91279e97..d9ebf7f3f34 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala
@@ -31,6 +31,7 @@ import org.apache.hudi.functional.ColumnStatIndexTestBase.ColumnStatsTestCase
import org.apache.hudi.storage.StoragePath
import org.apache.hudi.{ColumnStatsIndexSupport, DataSourceWriteOptions}
+import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
@@ -396,15 +397,14 @@ class TestColumnStatsIndex extends ColumnStatIndexTestBase {
val utils = new ParquetUtils
- val conf = HoodieTestUtils.getDefaultStorageConf
val path = new Path(pathStr)
- val fs = path.getFileSystem(conf.unwrap)
+ val storage = HoodieTestUtils.getStorage(new StoragePath(pathStr))
+ val fs = path.getFileSystem(storage.getConf.unwrapAs(classOf[Configuration]))
val parquetFilePath = new StoragePath(
fs.listStatus(path).filter(fs => fs.getPath.getName.endsWith(".parquet")).toSeq.head.getPath.toUri)
- val ranges = utils.readColumnStatsFromMetadata(conf, parquetFilePath,
- Seq("c1", "c2", "c3a", "c3b", "c3c", "c4", "c5", "c6", "c7", "c8").asJava)
+ val ranges = utils.readColumnStatsFromMetadata(storage, parquetFilePath, Seq("c1", "c2", "c3a", "c3b", "c3c", "c4", "c5", "c6", "c7", "c8").asJava)
ranges.asScala.foreach(r => {
// NOTE: Unfortunately Parquet can't compute statistics for Timestamp column, hence we
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkDataSource.scala
index 8c7e01488fc..04b37256a01 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkDataSource.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataTableWithSparkDataSource.scala
@@ -30,6 +30,7 @@ import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.metadata.{HoodieBackedTableMetadata, HoodieTableMetadata}
import org.apache.hudi.storage.StoragePath
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness.getSparkSqlConf
@@ -137,7 +138,8 @@ class TestMetadataTableWithSparkDataSource extends SparkClientFunctionalTestHarn
val partitionPathToTest = "2015/03/16"
val engineContext = new HoodieSparkEngineContext(jsc())
val metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexColumnStats(true).build();
- val baseTableMetada: HoodieTableMetadata = new HoodieBackedTableMetadata(engineContext, metadataConfig, s"$basePath", false)
+ val baseTableMetada: HoodieTableMetadata = new HoodieBackedTableMetadata(
+ engineContext, hoodieStorage(), metadataConfig, s"$basePath", false)
val fileStatuses = baseTableMetada.getAllFilesInPartition(new StoragePath(s"$basePath/" + partitionPathToTest))
val fileName = fileStatuses.get(0).getPath.getName
@@ -151,7 +153,8 @@ class TestMetadataTableWithSparkDataSource extends SparkClientFunctionalTestHarn
// read parquet file and verify stats
val colRangeMetadataList: java.util.List[HoodieColumnRangeMetadata[Comparable[_]]] = new ParquetUtils()
- .readColumnStatsFromMetadata(HadoopFSUtils.getStorageConf(jsc().hadoopConfiguration()),
+ .readColumnStatsFromMetadata(
+ new HoodieHadoopStorage(fileStatuses.get(0).getPath, HadoopFSUtils.getStorageConf(jsc().hadoopConfiguration())),
fileStatuses.get(0).getPath, Collections.singletonList("begin_lat"))
val columnRangeMetadata = colRangeMetadataList.get(0)
@@ -189,7 +192,8 @@ class TestMetadataTableWithSparkDataSource extends SparkClientFunctionalTestHarn
val partitionPathToTest = ""
val engineContext = new HoodieSparkEngineContext(jsc())
val metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexColumnStats(true).build();
- val baseTableMetada: HoodieTableMetadata = new HoodieBackedTableMetadata(engineContext, metadataConfig, s"$basePath", false)
+ val baseTableMetada: HoodieTableMetadata = new HoodieBackedTableMetadata(
+ engineContext, hoodieStorage(), metadataConfig, s"$basePath", false)
val allPartitionPaths = baseTableMetada.getAllPartitionPaths
assertEquals(allPartitionPaths.size(), 1)
@@ -207,7 +211,8 @@ class TestMetadataTableWithSparkDataSource extends SparkClientFunctionalTestHarn
// read parquet file and verify stats
val colRangeMetadataList: java.util.List[HoodieColumnRangeMetadata[Comparable[_]]] = new ParquetUtils()
- .readColumnStatsFromMetadata(HadoopFSUtils.getStorageConf(jsc().hadoopConfiguration()),
+ .readColumnStatsFromMetadata(
+ new HoodieHadoopStorage(fileStatuses.get(0).getPath, HadoopFSUtils.getStorageConf(jsc().hadoopConfiguration())),
fileStatuses.get(0).getPath, Collections.singletonList("begin_lat"))
val columnRangeMetadata = colRangeMetadataList.get(0)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
index e83270930f4..0b9794536cb 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
@@ -20,6 +20,11 @@ package org.apache.spark.sql.hudi.dml
import org.apache.hudi.HoodieSparkUtils
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+
+import java.util.function.Predicate
+import java.util.{Collections, List, Optional}
+import scala.collection.JavaConverters._
class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala
index f672f3068c3..bf6e222b763 100644
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
@@ -159,8 +160,9 @@ class Spark30LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala
index 74c75b0024d..aa1b798241c 100644
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
@@ -159,8 +160,9 @@ class Spark31LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32LegacyHoodieParquetFileFormat.scala
index 68f8ad2e30b..44d420c7501 100644
--- a/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32LegacyHoodieParquetFileFormat.scala
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
@@ -158,8 +159,9 @@ class Spark32LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala
index f2946b04d4e..d39d12b3fe2 100644
--- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala
@@ -29,6 +29,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
@@ -160,8 +161,9 @@ class Spark33LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala
index 9347f0024f2..8818cb5672f 100644
--- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapred.FileSplit
@@ -172,8 +173,8 @@ class Spark34LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
- InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
+ InternalSchemaCache.getInternalSchemaByVersionId(commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala
index 4ecdf451031..6286a19f080 100644
--- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala
@@ -27,6 +27,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.action.InternalSchemaMerger
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapred.FileSplit
@@ -173,8 +174,9 @@ class Spark35LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
val fileSchema = if (shouldUseInternalSchema) {
val commitInstantTime = FSUtils.getCommitTime(filePath.getName).toLong;
val validCommits = sharedConf.get(SparkInternalSchemaConverter.HOODIE_VALID_COMMITS_LIST)
+ val storage = new HoodieHadoopStorage(tablePath, broadcastedStorageConf.value)
InternalSchemaCache.getInternalSchemaByVersionId(
- commitInstantTime, tablePath, broadcastedStorageConf.value, if (validCommits == null) "" else validCommits)
+ commitInstantTime, tablePath, storage, if (validCommits == null) "" else validCommits)
} else {
null
}
diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
index 03085cc9d9b..3144d3d16f7 100644
--- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
+++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
@@ -123,6 +123,7 @@ public abstract class HoodieSyncClient implements HoodieMetaSyncOperations, Auto
public List<String> getAllPartitionPathsOnStorage() {
HoodieLocalEngineContext engineContext = new HoodieLocalEngineContext(metaClient.getStorageConf());
return FSUtils.getAllPartitionPaths(engineContext,
+ metaClient.getStorage(),
config.getString(META_SYNC_BASE_PATH),
config.getBoolean(META_SYNC_USE_FILE_LISTING_FROM_METADATA),
config.getBoolean(META_SYNC_ASSUME_DATE_PARTITION));
diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
index 20addf80d56..76ee1c8bce2 100644
--- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
+++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
@@ -107,7 +107,7 @@ public class ManifestFileWriter {
@VisibleForTesting
static Stream<String> getLatestBaseFiles(boolean canUseMetadataTable, HoodieEngineContext engContext, HoodieTableMetaClient metaClient,
boolean useAbsolutePath) {
- List<String> partitions = FSUtils.getAllPartitionPaths(engContext, metaClient.getBasePath(), canUseMetadataTable, false);
+ List<String> partitions = FSUtils.getAllPartitionPaths(engContext, metaClient.getStorage(), metaClient.getBasePath(), canUseMetadataTable, false);
LOG.info("Retrieve all partitions: " + partitions.size());
HoodieTableFileSystemView fsView = null;
try {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
index 6ae7507bec6..459483e547c 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
@@ -300,7 +300,7 @@ public class HoodieDataTableValidator implements Serializable {
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
try {
HoodieTableMetadata tableMetadata = new FileSystemBackedTableMetadata(
- engineContext, metaClient.getTableConfig(), engineContext.getStorageConf(), cfg.basePath, cfg.assumeDatePartitioning);
+ engineContext, metaClient.getTableConfig(), metaClient.getStorage(), cfg.basePath, cfg.assumeDatePartitioning);
List<StoragePath> allDataFilePaths = HoodieDataTableUtils.getBaseAndLogFilePathsFromFileSystem(tableMetadata, cfg.basePath);
// verify that no data files present with commit time < earliest commit in active timeline.
if (metaClient.getActiveTimeline().firstInstant().isPresent()) {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
index f856f35367c..a9bade03137 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
@@ -622,7 +622,7 @@ public class HoodieMetadataTableValidator implements Serializable {
List<String> allPartitionPathsFromFS = getPartitionsFromFileSystem(engineContext, basePath, metaClient.getStorage(),
completedTimeline);
- List<String> allPartitionPathsMeta = getPartitionsFromMDT(engineContext, basePath);
+ List<String> allPartitionPathsMeta = getPartitionsFromMDT(engineContext, basePath, metaClient.getStorage());
Collections.sort(allPartitionPathsFromFS);
Collections.sort(allPartitionPathsMeta);
@@ -675,14 +675,15 @@ public class HoodieMetadataTableValidator implements Serializable {
}
@VisibleForTesting
- List<String> getPartitionsFromMDT(HoodieEngineContext engineContext, String basePath) {
- return FSUtils.getAllPartitionPaths(engineContext, basePath, true, false);
+ List<String> getPartitionsFromMDT(HoodieEngineContext engineContext, String basePath,
+ HoodieStorage storage) {
+ return FSUtils.getAllPartitionPaths(engineContext, storage, basePath, true, false);
}
@VisibleForTesting
List<String> getPartitionsFromFileSystem(HoodieEngineContext engineContext, String basePath,
HoodieStorage storage, HoodieTimeline completedTimeline) {
- List<String> allPartitionPathsFromFS = FSUtils.getAllPartitionPaths(engineContext, basePath, false, false);
+ List<String> allPartitionPathsFromFS = FSUtils.getAllPartitionPaths(engineContext, storage, basePath, false, false);
// ignore partitions created by uncommitted ingestion.
return allPartitionPathsFromFS.stream().parallel().filter(part -> {
@@ -1393,7 +1394,8 @@ public class HoodieMetadataTableValidator implements Serializable {
.build();
this.fileSystemView = FileSystemViewManager.createInMemoryFileSystemView(engineContext,
metaClient, metadataConfig);
- this.tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, metaClient.getBasePathV2().toString());
+ this.tableMetadata = HoodieTableMetadata.create(
+ engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePathV2().toString());
if (metaClient.getCommitsTimeline().filterCompletedInstants().countInstants() > 0) {
this.allColumnNameList = getAllColumnNames();
}
@@ -1440,7 +1442,7 @@ public class HoodieMetadataTableValidator implements Serializable {
} else {
return baseFileNameList.stream().flatMap(filename ->
FileFormatUtils.getInstance(HoodieFileFormat.PARQUET).readColumnStatsFromMetadata(
- metaClient.getStorageConf(),
+ metaClient.getStorage(),
new StoragePath(FSUtils.constructAbsolutePath(metaClient.getBasePathV2(), partitionPath), filename),
allColumnNameList).stream())
.sorted(new HoodieColumnRangeMetadataComparator())
@@ -1488,7 +1490,7 @@ public class HoodieMetadataTableValidator implements Serializable {
HoodieConfig hoodieConfig = new HoodieConfig();
hoodieConfig.setValue(HoodieReaderConfig.USE_NATIVE_HFILE_READER,
Boolean.toString(ConfigUtils.getBooleanWithAltKeys(props, HoodieReaderConfig.USE_NATIVE_HFILE_READER)));
- try (HoodieFileReader fileReader = getHoodieSparkIOFactory(metaClient.getStorageConf())
+ try (HoodieFileReader fileReader = getHoodieSparkIOFactory(metaClient.getStorage())
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, path)) {
bloomFilter = fileReader.readBloomFilter();
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java
index f7fdbcae64c..b2bb34ede3b 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieRepairTool.java
@@ -167,7 +167,7 @@ public class HoodieRepairTool {
.build();
this.tableMetadata = new FileSystemBackedTableMetadata(
- context, metaClient.getTableConfig(), context.getStorageConf(), cfg.basePath, cfg.assumeDatePartitioning);
+ context, metaClient.getTableConfig(), metaClient.getStorage(), cfg.basePath, cfg.assumeDatePartitioning);
}
public boolean run() {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
index 9b3dcc6ffe1..60113c08b3a 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
@@ -36,6 +36,7 @@ import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
@@ -105,7 +106,7 @@ public class HoodieSnapshotCopier implements Serializable {
LOG.info(String.format("Starting to snapshot latest version files which are also no-late-than %s.",
latestCommitTimestamp));
- List<String> partitions = FSUtils.getAllPartitionPaths(context, baseDir, useFileListingFromMetadata, shouldAssumeDatePartitioning);
+ List<String> partitions = FSUtils.getAllPartitionPaths(context, new HoodieHadoopStorage(fs), baseDir, useFileListingFromMetadata, shouldAssumeDatePartitioning);
if (partitions.size() > 0) {
LOG.info(String.format("The job needs to copy %d partitions.", partitions.size()));
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
index c6c8a393bbd..00463433500 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
@@ -40,6 +40,7 @@ import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.util.JavaScalaConverters;
import org.apache.hudi.utilities.exception.HoodieSnapshotExporterException;
@@ -137,7 +138,7 @@ public class HoodieSnapshotExporter {
latestCommitTimestamp));
final HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
- final List<String> partitions = getPartitions(engineContext, cfg);
+ final List<String> partitions = getPartitions(engineContext, cfg, new HoodieHadoopStorage(sourceFs));
if (partitions.isEmpty()) {
throw new HoodieSnapshotExporterException("The source dataset has 0 partition to snapshot.");
}
@@ -160,8 +161,8 @@ public class HoodieSnapshotExporter {
return latestCommit.isPresent() ? Option.of(latestCommit.get().getTimestamp()) : Option.empty();
}
- private List<String> getPartitions(HoodieEngineContext engineContext, Config cfg) {
- return FSUtils.getAllPartitionPaths(engineContext, cfg.sourceBasePath, true, false);
+ private List<String> getPartitions(HoodieEngineContext engineContext, Config cfg, HoodieStorage storage) {
+ return FSUtils.getAllPartitionPaths(engineContext, storage, cfg.sourceBasePath, true, false);
}
private void createSuccessTag(FileSystem fs, Config cfg) throws IOException {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
index 1a6a1ba4f82..ff655dfd017 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
@@ -35,6 +35,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
@@ -276,8 +277,9 @@ public class TableSizeStats implements Serializable {
.enable(isMetadataEnabled(basePath, jsc))
.build();
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
- HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, basePath);
StorageConfiguration<?> storageConf = HadoopFSUtils.getStorageConfWithCopy(jsc.hadoopConfiguration());
+ HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(
+ engineContext, new HoodieHadoopStorage(basePath, storageConf), metadataConfig, basePath);
List<String> allPartitions = tableMetadata.getAllPartitionPaths();
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java
index 1dd008da237..9ad99798663 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java
@@ -21,7 +21,7 @@ package org.apache.hudi.utilities.deltastreamer;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.utilities.streamer.HoodieStreamerMetrics;
/**
@@ -31,11 +31,11 @@ import org.apache.hudi.utilities.streamer.HoodieStreamerMetrics;
@Deprecated
public class HoodieDeltaStreamerMetrics extends HoodieStreamerMetrics {
- public HoodieDeltaStreamerMetrics(HoodieWriteConfig writeConfig, StorageConfiguration<?> storageConf) {
- super(writeConfig.getMetricsConfig(), storageConf);
+ public HoodieDeltaStreamerMetrics(HoodieWriteConfig writeConfig, HoodieStorage storage) {
+ super(writeConfig.getMetricsConfig(), storage);
}
- public HoodieDeltaStreamerMetrics(HoodieMetricsConfig metricsConfig, StorageConfiguration<?> storageConf) {
- super(metricsConfig, storageConf);
+ public HoodieDeltaStreamerMetrics(HoodieMetricsConfig metricsConfig, HoodieStorage storage) {
+ super(metricsConfig, storage);
}
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java
index 378ba45e3e9..632f8bbafb3 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java
@@ -20,7 +20,7 @@ package org.apache.hudi.utilities.ingestion;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import com.codahale.metrics.Timer;
@@ -31,17 +31,17 @@ import java.io.Serializable;
*/
public abstract class HoodieIngestionMetrics implements Serializable {
- protected final StorageConfiguration<?> storageConf;
+ protected final HoodieStorage storage;
protected final HoodieMetricsConfig writeConfig;
- public HoodieIngestionMetrics(HoodieWriteConfig writeConfig, StorageConfiguration<?> storageConf) {
- this(writeConfig.getMetricsConfig(), storageConf);
+ public HoodieIngestionMetrics(HoodieWriteConfig writeConfig, HoodieStorage storage) {
+ this(writeConfig.getMetricsConfig(), storage);
}
- public HoodieIngestionMetrics(HoodieMetricsConfig writeConfig, StorageConfiguration<?> storageConf) {
+ public HoodieIngestionMetrics(HoodieMetricsConfig writeConfig, HoodieStorage storage) {
this.writeConfig = writeConfig;
- this.storageConf = storageConf;
+ this.storage = storage;
}
public abstract Timer.Context getOverallTimerContext();
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java
index ac6b1a90b31..13f0141d481 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java
@@ -34,6 +34,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.timeline.service.TimelineService;
import org.apache.hudi.utilities.UtilHelpers;
@@ -100,7 +101,9 @@ public class TimelineServerPerf implements Serializable {
public void run() throws IOException {
JavaSparkContext jsc = UtilHelpers.buildSparkContext("hudi-view-perf-" + cfg.basePath, cfg.sparkMaster);
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
- List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(engineContext, cfg.basePath, cfg.useFileListingFromMetadata, true);
+ List<String> allPartitionPaths = FSUtils.getAllPartitionPaths(
+ engineContext, new HoodieHadoopStorage(cfg.basePath, engineContext.getStorageConf()),
+ cfg.basePath, cfg.useFileListingFromMetadata, true);
Collections.shuffle(allPartitionPaths);
List<String> selected = allPartitionPaths.stream().filter(p -> !p.contains("error")).limit(cfg.maxPartitions)
.collect(Collectors.toList());
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java
index c5c01bee231..41462dc4198 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerMetrics.java
@@ -22,7 +22,7 @@ package org.apache.hudi.utilities.streamer;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.metrics.Metrics;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.utilities.ingestion.HoodieIngestionMetrics;
import com.codahale.metrics.Timer;
@@ -38,14 +38,14 @@ public class HoodieStreamerMetrics extends HoodieIngestionMetrics {
private transient Timer hiveSyncTimer;
private transient Timer metaSyncTimer;
- public HoodieStreamerMetrics(HoodieWriteConfig writeConfig, StorageConfiguration<?> storageConf) {
- this(writeConfig.getMetricsConfig(), storageConf);
+ public HoodieStreamerMetrics(HoodieWriteConfig writeConfig, HoodieStorage storage) {
+ this(writeConfig.getMetricsConfig(), storage);
}
- public HoodieStreamerMetrics(HoodieMetricsConfig writeConfig, StorageConfiguration<?> storageConf) {
- super(writeConfig, storageConf);
+ public HoodieStreamerMetrics(HoodieMetricsConfig writeConfig, HoodieStorage storage) {
+ super(writeConfig, storage);
if (writeConfig.isMetricsOn()) {
- metrics = Metrics.getInstance(writeConfig, storageConf);
+ metrics = Metrics.getInstance(writeConfig, storage);
this.overallTimerName = getMetricsName("timer", "deltastreamer");
this.hiveSyncTimerName = getMetricsName("timer", "deltastreamerHiveSync");
this.metaSyncTimerName = getMetricsName("timer", "deltastreamerMetaSync");
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
index d5405645782..dcde6eb8d4f 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
@@ -75,7 +75,6 @@ import org.apache.hudi.keygen.KeyGenUtils;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.apache.hudi.sync.common.util.SyncUtilHelpers;
@@ -314,9 +313,9 @@ public class StreamSync implements Serializable, Closeable {
HoodieWriteConfig hoodieWriteConfig = getHoodieClientConfig();
this.metrics = (HoodieIngestionMetrics) ReflectionUtils.loadClass(cfg.ingestionMetricsClass,
- new Class<?>[] { HoodieMetricsConfig.class, StorageConfiguration.class},
- hoodieWriteConfig.getMetricsConfig(), storage.getConf());
- this.hoodieMetrics = new HoodieMetrics(hoodieWriteConfig, storage.getConf());
+ new Class<?>[] {HoodieMetricsConfig.class, HoodieStorage.class},
+ hoodieWriteConfig.getMetricsConfig(), storage);
+ this.hoodieMetrics = new HoodieMetrics(hoodieWriteConfig, storage);
if (props.getBoolean(ERROR_TABLE_ENABLED.key(), ERROR_TABLE_ENABLED.defaultValue())) {
this.errorTableWriter = ErrorTableUtils.getErrorTableWriter(
cfg, sparkSession, props, hoodieSparkContext, fs);
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
index 0f399134047..00e8cc5bc76 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
@@ -205,7 +205,7 @@ public class TestHoodieIndexer extends SparkClientFunctionalTestHarness implemen
metaClient.getActiveTimeline().readIndexPlanAsBytes(indexingInstant).get());
String indexUptoInstantTime = indexPlan.getIndexPartitionInfos().get(0).getIndexUptoInstant();
HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(
- context(), metadataConfig, metaClient.getBasePathV2().toString());
+ context(), metaClient.getStorage(), metadataConfig, metaClient.getBasePathV2().toString());
HoodieTableMetaClient metadataMetaClient = metadata.getMetadataMetaClient();
String mdtCommitTime = HoodieTableMetadataUtil.createAsyncIndexerTimestamp(indexUptoInstantTime);
assertTrue(metadataMetaClient.getActiveTimeline().containsInstant(mdtCommitTime));
@@ -270,7 +270,7 @@ public class TestHoodieIndexer extends SparkClientFunctionalTestHarness implemen
metaClient.getActiveTimeline().revertToInflight(commit);
HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(
- context(), metadataConfig, metaClient.getBasePathV2().toString());
+ context(), metaClient.getStorage(), metadataConfig, metaClient.getBasePathV2().toString());
HoodieTableMetaClient metadataMetaClient = metadata.getMetadataMetaClient();
HoodieInstant mdtCommit = metadataMetaClient.getActiveTimeline()
.filter(i -> i.getTimestamp().equals(commitTime))
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
index a9af0146db1..6230c5815f6 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
@@ -198,7 +198,7 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
}
@Override
- List<String> getPartitionsFromMDT(HoodieEngineContext engineContext, String basePath) {
+ List<String> getPartitionsFromMDT(HoodieEngineContext engineContext, String basePath, HoodieStorage storage) {
return metadataPartitionsToReturn;
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index 4da6ef51b62..64efb663fcb 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -2910,7 +2910,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
syncAndAssertRecordCount(cfg, 1000, tableBasePath, "00000", 1);
HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setBasePath(tableBasePath).setConf(HoodieTestUtils.getDefaultStorageConf()).build();
- List<String> partitions = FSUtils.getAllPartitionPaths(new HoodieLocalEngineContext(metaClient.getStorageConf()), metaClient.getBasePath(), false, false);
+ List<String> partitions = FSUtils.getAllPartitionPaths(new HoodieLocalEngineContext(metaClient.getStorageConf()), metaClient.getStorage(), metaClient.getBasePath(), false, false);
StorageConfiguration hadoopConf = metaClient.getStorageConf();
HoodieLocalEngineContext engContext = new HoodieLocalEngineContext(hadoopConf);
HoodieMetadataFileSystemView fsView = new HoodieMetadataFileSystemView(engContext, metaClient,
@@ -2921,7 +2921,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
assertEquals(baseFiles.size(), partitions.size());
// Verify if each parquet file is actually sorted by sortColumn.
for (String filePath : baseFiles) {
- try (HoodieAvroParquetReader parquetReader = new HoodieAvroParquetReader(HoodieTestUtils.getDefaultStorageConf(), new StoragePath(filePath))) {
+ try (HoodieAvroParquetReader parquetReader = new HoodieAvroParquetReader(HoodieTestUtils.getStorage(filePath), new StoragePath(filePath))) {
ClosableIterator<HoodieRecord<IndexedRecord>> iterator = parquetReader.getRecordIterator();
List<Float> sortColumnValues = new ArrayList<>();
while (iterator.hasNext()) {
(hudi) 02/07: [HUDI-7775] Remove unused APIs in HoodieStorage (#11281)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch release-0.15.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 670f13102f05f077adfc29892cbd199132c335ba
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Fri May 24 09:29:53 2024 -0700
[HUDI-7775] Remove unused APIs in HoodieStorage (#11281)
---
.../java/org/apache/hudi/common/util/OrcUtils.java | 2 +-
.../hudi/storage/hadoop/HoodieHadoopStorage.java | 11 ---------
.../org/apache/hudi/storage/HoodieStorage.java | 27 ----------------------
.../hudi/io/storage/TestHoodieStorageBase.java | 9 --------
4 files changed, 1 insertion(+), 48 deletions(-)
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
index 8727ca5041d..03d72f4a97b 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
@@ -294,7 +294,7 @@ public class OrcUtils extends FileFormatUtils {
// Since we are only interested in saving metadata to the footer, the schema, blocksizes and other
// parameters are not important.
Schema schema = HoodieAvroUtils.getRecordKeySchema();
- OrcFile.WriterOptions writerOptions = OrcFile.writerOptions(storage.unwrapConfAs(Configuration.class))
+ OrcFile.WriterOptions writerOptions = OrcFile.writerOptions(storage.getConf().unwrapAs(Configuration.class))
.fileSystem((FileSystem) storage.getFileSystem())
.setSchema(AvroOrcUtils.createOrcSchema(schema));
try (Writer writer = OrcFile.createWriter(convertToHadoopPath(filePath), writerOptions)) {
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
index 72262f6b5d4..f7df5e1f164 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
@@ -220,12 +220,6 @@ public class HoodieHadoopStorage extends HoodieStorage {
return fs.delete(convertToHadoopPath(path), false);
}
- @Override
- public StoragePath makeQualified(StoragePath path) {
- return convertToStoragePath(
- fs.makeQualified(convertToHadoopPath(path)));
- }
-
@Override
public Object getFileSystem() {
return fs;
@@ -236,11 +230,6 @@ public class HoodieHadoopStorage extends HoodieStorage {
return new HadoopStorageConfiguration(fs.getConf());
}
- @Override
- public Configuration unwrapConf() {
- return fs.getConf();
- }
-
@Override
public HoodieStorage getRawStorage() {
if (fs instanceof HoodieWrapperFileSystem) {
diff --git a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
index fcc8d2d505d..a92a2938bfc 100644
--- a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
+++ b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
@@ -39,8 +39,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
-import static org.apache.hudi.storage.StorageConfiguration.castConfiguration;
-
/**
* Provides I/O APIs on files and directories on storage.
* The APIs are mainly based on {@code org.apache.hadoop.fs.FileSystem} class.
@@ -252,15 +250,6 @@ public abstract class HoodieStorage implements Closeable {
@PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
public abstract boolean deleteFile(StoragePath path) throws IOException;
- /**
- * Qualifies a path to one which uses this storage and, if relative, made absolute.
- *
- * @param path to qualify.
- * @return Qualified path.
- */
- @PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
- public abstract StoragePath makeQualified(StoragePath path);
-
/**
* @return the underlying file system instance if exists.
*/
@@ -273,12 +262,6 @@ public abstract class HoodieStorage implements Closeable {
@PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
public abstract StorageConfiguration<?> getConf();
- /**
- * @return the underlying configuration instance.
- */
- @PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
- public abstract Object unwrapConf();
-
/**
* @return the raw storage.
*/
@@ -437,14 +420,4 @@ public abstract class HoodieStorage implements Closeable {
public List<StoragePathInfo> globEntries(StoragePath pathPattern) throws IOException {
return globEntries(pathPattern, e -> true);
}
-
- /**
- * @param clazz class of U.
- * @param <U> type to return.
- * @return the underlying configuration cast to type {@link U}.
- */
- @PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
- public final <U> U unwrapConfAs(Class<U> clazz) {
- return castConfiguration(unwrapConf(), clazz);
- }
}
diff --git a/hudi-io/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageBase.java b/hudi-io/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageBase.java
index cdc8d6f6746..c953fbab7a9 100644
--- a/hudi-io/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageBase.java
+++ b/hudi-io/src/test/java/org/apache/hudi/io/storage/TestHoodieStorageBase.java
@@ -324,15 +324,6 @@ public abstract class TestHoodieStorageBase {
assertFalse(storage.deleteDirectory(path2));
}
- @Test
- public void testMakeQualified() {
- HoodieStorage storage = getStorage();
- StoragePath path = new StoragePath("/tmp/testMakeQualified/1.file");
- assertEquals(
- new StoragePath("file:/tmp/testMakeQualified/1.file"),
- storage.makeQualified(path));
- }
-
@Test
public void testGetFileSystem() {
Object conf = getConf();