You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/04/18 17:20:27 UTC
(hudi) branch master updated: [HUDI-6497] Replace FileSystem, Path, and FileStatus usage in hudi-common module (#10591)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 8a0ce61b2fd [HUDI-6497] Replace FileSystem, Path, and FileStatus usage in hudi-common module (#10591)
8a0ce61b2fd is described below
commit 8a0ce61b2fd187243e89254cdef528b3b1cb49e8
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Thu Apr 18 10:20:20 2024 -0700
[HUDI-6497] Replace FileSystem, Path, and FileStatus usage in hudi-common module (#10591)
This commit makes the changes to replace most `FileSystem`, `Path`, and `FileStatus` usage with `HoodieStorage`, `StoragePath` and `StoragePathInfo` (introduced in #10567, renamed in #10672) in `hudi-common` module, to remove dependency on Hadoop FS abstraction which is not essential to most Hudi core read and write logic.
This commit still keeps using the Hadoop FileSystem-based implementation under the hood. A follow-up PR will make `HoodieStorage` and I/O implementation pluggable.
---
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 17 +-
.../main/java/org/apache/hudi/cli/HoodieCLI.java | 10 +-
.../hudi/cli/commands/ArchivedCommitsCommand.java | 33 +-
.../hudi/cli/commands/CompactionCommand.java | 42 +-
.../apache/hudi/cli/commands/ExportCommand.java | 34 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 18 +-
.../hudi/cli/commands/HoodieLogFileCommand.java | 33 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 108 ++--
.../apache/hudi/cli/commands/RepairsCommand.java | 56 ++-
.../org/apache/hudi/cli/commands/SparkMain.java | 10 +-
.../org/apache/hudi/cli/commands/StatsCommand.java | 27 +-
.../org/apache/hudi/cli/commands/TableCommand.java | 23 +-
.../apache/hudi/cli/commands/TimelineCommand.java | 33 +-
.../cli/commands/TestArchivedCommitsCommand.java | 3 +-
.../hudi/cli/commands/TestCleansCommand.java | 9 +-
.../hudi/cli/commands/TestCommitsCommand.java | 13 +-
.../hudi/cli/commands/TestCompactionCommand.java | 7 +-
.../apache/hudi/cli/commands/TestDiffCommand.java | 9 +-
.../cli/commands/TestHoodieLogFileCommand.java | 29 +-
.../hudi/cli/commands/TestRepairsCommand.java | 5 +-
.../cli/commands/TestUpgradeDowngradeCommand.java | 19 +-
.../cli/integ/ITTestHDFSParquetImportCommand.java | 20 +-
.../hudi/cli/integ/ITTestRepairsCommand.java | 60 ++-
.../hudi/cli/integ/ITTestSavepointsCommand.java | 6 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 15 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 4 +-
.../apache/hudi/client/CompactionAdminClient.java | 48 +-
.../client/embedded/EmbeddedTimelineService.java | 8 +-
.../hudi/client/heartbeat/HeartbeatUtils.java | 38 +-
.../client/heartbeat/HoodieHeartbeatClient.java | 24 +-
.../client/timeline/HoodieTimelineArchiver.java | 2 +-
.../hudi/client/timeline/LSMTimelineWriter.java | 39 +-
.../client/transaction/TransactionManager.java | 5 +-
.../utils/LegacyArchivedMetaEntryReader.java | 42 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 5 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 36 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 8 +-
.../java/org/apache/hudi/io/HoodieCDCLogger.java | 2 +-
.../org/apache/hudi/io/HoodieConcatHandle.java | 3 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 20 +-
.../java/org/apache/hudi/io/HoodieIOHandle.java | 7 +-
.../hudi/io/HoodieKeyLocationFetchHandle.java | 7 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 18 +-
.../org/apache/hudi/io/HoodieMergedReadHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieReadHandle.java | 13 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 27 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 45 +-
.../java/org/apache/hudi/table/HoodieTable.java | 23 +-
.../table/action/clean/CleanActionExecutor.java | 7 +-
.../hudi/table/action/clean/CleanPlanner.java | 7 +-
.../action/commit/BaseCommitActionExecutor.java | 3 +-
.../table/action/commit/HoodieMergeHelper.java | 5 +-
.../hudi/table/action/compact/HoodieCompactor.java | 13 +-
.../HoodieLogCompactionPlanGenerator.java | 2 +-
.../table/action/index/RunIndexActionExecutor.java | 7 +-
.../action/index/ScheduleIndexActionExecutor.java | 2 +-
.../BaseHoodieFunctionalIndexClient.java | 4 +-
.../action/restore/BaseRestoreActionExecutor.java | 2 +-
.../rollback/BaseRollbackActionExecutor.java | 10 +-
.../table/action/rollback/BaseRollbackHelper.java | 24 +-
.../rollback/ListingBasedRollbackStrategy.java | 28 +-
.../hudi/table/action/rollback/RollbackUtils.java | 4 +-
.../ttl/strategy/KeepByCreationTimeStrategy.java | 2 +-
.../hudi/table/marker/DirectWriteMarkers.java | 56 ++-
.../table/marker/MarkerBasedRollbackUtils.java | 18 +-
.../SimpleDirectMarkerBasedDetectionStrategy.java | 6 +-
...nsactionDirectMarkerBasedDetectionStrategy.java | 10 +-
.../marker/TimelineServerBasedWriteMarkers.java | 12 +-
.../org/apache/hudi/table/marker/WriteMarkers.java | 36 +-
.../org/apache/hudi/table/repair/RepairUtils.java | 10 +-
.../table/upgrade/FiveToSixUpgradeHandler.java | 8 +-
.../table/upgrade/SixToFiveDowngradeHandler.java | 3 +-
.../table/upgrade/TwoToOneDowngradeHandler.java | 33 +-
.../hudi/table/upgrade/UpgradeDowngrade.java | 9 +-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 11 +-
.../hudi/avro/TestHoodieAvroParquetWriter.java | 16 +-
.../heartbeat/TestHoodieHeartbeatClient.java | 35 +-
.../timeline/TestCompletionTimeQueryView.java | 8 +-
.../hudi/client/transaction/TestLockManager.java | 4 +-
.../client/transaction/TestTransactionManager.java | 5 +-
.../utils/TestLegacyArchivedMetaEntryReader.java | 6 +-
.../common/testutils/HoodieMetadataTestTable.java | 17 +-
.../hudi/table/marker/TestWriteMarkersFactory.java | 8 +-
.../apache/hudi/table/repair/TestRepairUtils.java | 19 +-
.../GenericRecordValidationTestUtils.java | 4 +-
.../hudi/testutils/HoodieWriteableTestTable.java | 43 +-
.../hudi/testutils/providers/DFSProvider.java | 5 +-
.../providers/HoodieMetaClientProvider.java | 8 +-
.../hudi/utils/HoodieWriterClientTestHarness.java | 27 +-
.../java/org/apache/hudi/io/FlinkAppendHandle.java | 4 +-
.../hudi/io/FlinkConcatAndReplaceHandle.java | 10 +-
.../java/org/apache/hudi/io/FlinkConcatHandle.java | 6 +-
.../java/org/apache/hudi/io/FlinkCreateHandle.java | 22 +-
.../apache/hudi/io/FlinkMergeAndReplaceHandle.java | 27 +-
.../FlinkMergeAndReplaceHandleWithChangeLog.java | 7 +-
.../java/org/apache/hudi/io/FlinkMergeHandle.java | 36 +-
.../hudi/io/FlinkMergeHandleWithChangeLog.java | 3 +-
.../apache/hudi/io/FlinkWriteHandleFactory.java | 15 +-
.../java/org/apache/hudi/io/MiniBatchHandle.java | 4 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 23 +-
.../row/HoodieRowDataFileWriterFactory.java | 3 +-
.../io/storage/row/HoodieRowDataParquetWriter.java | 7 +-
.../hudi/table/HoodieFlinkCopyOnWriteTable.java | 1 +
.../FlinkDeletePartitionCommitActionExecutor.java | 24 +-
.../index/bloom/TestFlinkHoodieBloomIndex.java | 4 +-
.../testutils/HoodieFlinkWriteableTestTable.java | 26 +-
.../run/strategy/JavaExecutionStrategy.java | 8 +-
.../hudi/table/HoodieJavaCopyOnWriteTable.java | 1 +
.../org/apache/hudi/table/HoodieJavaTable.java | 5 +-
.../commit/BaseJavaCommitActionExecutor.java | 5 +-
.../client/TestHoodieJavaWriteClientInsert.java | 9 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 315 +++++++-----
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 58 ++-
.../commit/TestJavaCopyOnWriteActionExecutor.java | 23 +-
.../testutils/HoodieJavaClientTestHarness.java | 108 ++--
.../hudi/testutils/TestHoodieMetadataBase.java | 4 +-
.../MultipleSparkJobExecutionStrategy.java | 25 +-
.../strategy/SingleSparkJobExecutionStrategy.java | 4 +-
.../client/utils/SparkMetadataWriterUtils.java | 22 +-
.../index/bloom/HoodieFileProbingFunction.java | 4 +-
.../index/bloom/SparkHoodieBloomIndexHelper.java | 12 +-
.../io/storage/HoodieSparkFileReaderFactory.java | 8 +-
.../io/storage/HoodieSparkFileWriterFactory.java | 12 +-
.../hudi/io/storage/HoodieSparkParquetReader.java | 7 +-
.../hudi/io/storage/HoodieSparkParquetWriter.java | 5 +-
.../row/HoodieInternalRowFileWriterFactory.java | 7 +-
.../row/HoodieInternalRowParquetWriter.java | 8 +-
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 22 +-
.../org/apache/hudi/table/HoodieSparkTable.java | 5 +-
.../bootstrap/BaseBootstrapMetadataHandler.java | 11 +-
.../bootstrap/OrcBootstrapMetadataHandler.java | 12 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 10 +-
.../SparkDeletePartitionCommitActionExecutor.java | 27 +-
.../hudi/BaseSparkInternalRowReaderContext.java | 8 +-
.../SparkFileFormatInternalRowReaderContext.scala | 13 +-
.../scala/org/apache/hudi/util/PathUtils.scala | 28 +-
.../HoodieSparkPartitionedFileUtils.scala | 6 +-
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 9 +-
.../org/apache/hudi/client/TestClientRollback.java | 9 +-
.../hudi/client/TestHoodieClientMultiWriter.java | 23 +-
.../TestMultiWriterWithPreferWriterIngestion.java | 9 +-
...erBasedDetectionStrategyWithZKLockProvider.java | 2 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 7 +-
...parkConsistentBucketClusteringPlanStrategy.java | 4 +-
.../functional/TestConsistentBucketIndex.java | 4 +-
.../functional/TestHoodieBackedMetadata.java | 448 ++++++++++-------
.../functional/TestHoodieBackedTableMetadata.java | 46 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 139 ++---
.../TestHoodieClientOnMergeOnReadStorage.java | 4 +-
.../hudi/client/functional/TestHoodieIndex.java | 28 +-
.../client/functional/TestHoodieMetadataBase.java | 6 +-
.../TestRemoteFileSystemViewWithMetadataTable.java | 6 +-
.../TestSavepointRestoreMergeOnRead.java | 23 +-
.../TestRDDSimpleBucketBulkInsertPartitioner.java | 3 +-
.../index/bloom/TestBloomIndexTagWithColStats.java | 9 +-
.../hudi/index/bloom/TestHoodieBloomIndex.java | 76 +--
.../index/bloom/TestHoodieGlobalBloomIndex.java | 35 +-
.../index/bucket/TestHoodieSimpleBucketIndex.java | 2 +-
.../index/hbase/TestSparkHoodieHBaseIndex.java | 5 +-
.../hudi/io/TestHoodieKeyLocationFetchHandle.java | 4 +-
.../org/apache/hudi/io/TestHoodieMergeHandle.java | 7 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 117 +++--
.../storage/TestHoodieAvroFileWriterFactory.java | 16 +-
.../java/org/apache/hudi/table/TestCleaner.java | 13 +-
.../apache/hudi/table/TestConsistencyGuard.java | 51 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 22 +-
.../table/action/bootstrap/TestBootstrapUtils.java | 25 +-
.../commit/TestCopyOnWriteActionExecutor.java | 69 ++-
.../table/action/compact/CompactionTestBase.java | 8 +-
.../table/action/compact/TestAsyncCompaction.java | 16 +-
.../table/action/compact/TestHoodieCompactor.java | 4 +-
.../strategy/TestHoodieCompactionStrategy.java | 3 +-
.../rollback/HoodieClientRollbackTestBase.java | 14 +-
.../TestCopyOnWriteRollbackActionExecutor.java | 45 +-
.../TestMergeOnReadRollbackActionExecutor.java | 13 +-
.../table/action/rollback/TestRollbackUtils.java | 19 +-
...dieSparkMergeOnReadTableInsertUpdateDelete.java | 34 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 86 ++--
.../TestMarkerBasedRollbackStrategy.java | 2 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 14 +-
.../TestTimelineServerBasedWriteMarkers.java | 19 +-
.../hudi/table/marker/TestWriteMarkersBase.java | 18 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 69 ++-
.../hudi/testutils/FunctionalTestHarness.java | 35 +-
.../hudi/testutils/HoodieCleanerTestBase.java | 5 +-
.../hudi/testutils/HoodieClientTestBase.java | 44 +-
.../hudi/testutils/HoodieClientTestUtils.java | 34 +-
.../testutils/HoodieSparkClientTestHarness.java | 90 ++--
.../testutils/HoodieSparkWriteableTestTable.java | 32 +-
.../SparkClientFunctionalTestHarness.java | 47 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 59 ++-
.../org/apache/hudi/common/HoodieRollbackStat.java | 14 +-
.../hudi/common/bootstrap/FileStatusUtils.java | 14 +-
.../bootstrap/index/HFileBootstrapIndex.java | 72 +--
.../common/config/DFSPropertiesConfiguration.java | 43 +-
.../common/config/HoodieFunctionalIndexConfig.java | 65 +--
.../DirectMarkerBasedDetectionStrategy.java | 32 +-
.../TimelineServerBasedDetectionStrategy.java | 7 +-
.../hudi/common/engine/HoodieReaderContext.java | 13 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 314 ++++++++----
.../hudi/common/fs/FailSafeConsistencyGuard.java | 51 +-
.../hudi/common/fs/OptimisticConsistencyGuard.java | 19 +-
.../common/heartbeat/HoodieHeartbeatUtils.java | 25 +-
.../org/apache/hudi/common/model/BaseFile.java | 39 +-
.../common/model/BootstrapBaseFileMapping.java | 2 +-
.../hudi/common/model/CompactionOperation.java | 11 +-
.../hudi/common/model/HoodieArchivedLogFile.java | 10 +-
.../apache/hudi/common/model/HoodieBaseFile.java | 50 +-
.../hudi/common/model/HoodieCommitMetadata.java | 50 +-
.../apache/hudi/common/model/HoodieLogFile.java | 48 +-
.../hudi/common/model/HoodiePartitionMetadata.java | 81 +--
.../apache/hudi/common/model/HoodieWriteStat.java | 5 +-
.../hudi/common/table/HoodieTableConfig.java | 65 +--
.../hudi/common/table/HoodieTableMetaClient.java | 194 ++++---
.../hudi/common/table/TableSchemaResolver.java | 24 +-
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 45 +-
.../table/log/AbstractHoodieLogRecordReader.java | 26 +-
.../table/log/BaseHoodieLogRecordReader.java | 33 +-
.../table/log/HoodieCDCLogRecordIterator.java | 10 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 62 +--
.../hudi/common/table/log/HoodieLogFormat.java | 36 +-
.../common/table/log/HoodieLogFormatReader.java | 12 +-
.../table/log/HoodieLogFormatReverseReader.java | 15 +-
.../common/table/log/HoodieLogFormatWriter.java | 24 +-
.../table/log/HoodieMergedLogRecordReader.java | 19 +-
.../table/log/HoodieMergedLogRecordScanner.java | 19 +-
.../table/log/HoodieUnMergedLogRecordScanner.java | 14 +-
.../hudi/common/table/log/LogReaderUtils.java | 14 +-
.../table/log/block/HoodieHFileDataBlock.java | 22 +-
.../table/log/block/HoodieParquetDataBlock.java | 14 +-
.../common/table/read/HoodieFileGroupReader.java | 50 +-
.../table/timeline/HoodieActiveTimeline.java | 120 +++--
.../table/timeline/HoodieArchivedTimeline.java | 7 +-
.../hudi/common/table/timeline/HoodieInstant.java | 7 +-
.../hudi/common/table/timeline/LSMTimeline.java | 44 +-
.../table/timeline/TimelineMetadataUtils.java | 4 +-
.../common/table/timeline/dto/BaseFileDTO.java | 11 +-
.../common/table/timeline/dto/FilePathDTO.java | 9 +-
.../common/table/timeline/dto/FileStatusDTO.java | 60 +--
.../common/table/timeline/dto/InstantStateDTO.java | 8 +-
.../hudi/common/table/timeline/dto/LogFileDTO.java | 8 +-
.../clean/CleanPlanV2MigrationHandler.java | 2 +-
.../table/view/AbstractTableFileSystemView.java | 129 ++---
.../table/view/HoodieTableFileSystemView.java | 10 +-
.../IncrementalTimelineSyncFileSystemView.java | 38 +-
.../view/RemoteHoodieTableFileSystemView.java | 8 +-
.../table/view/RocksDbBasedFileSystemView.java | 12 +-
.../view/SpillableMapBasedFileSystemView.java | 11 +-
.../org/apache/hudi/common/util/BaseFileUtils.java | 131 ++---
.../org/apache/hudi/common/util/ConfigUtils.java | 28 +-
.../hudi/common/util/InternalSchemaCache.java | 41 +-
.../org/apache/hudi/common/util/MarkerUtils.java | 111 ++--
.../java/org/apache/hudi/common/util/OrcUtils.java | 56 ++-
.../org/apache/hudi/common/util/ParquetUtils.java | 91 ++--
.../apache/hudi/common/util/TablePathUtils.java | 45 +-
.../hudi/exception/InvalidHoodiePathException.java | 8 +-
.../hudi/exception/TableNotFoundException.java | 12 +-
.../index/secondary/SecondaryIndexManager.java | 8 +-
.../io/FileBasedInternalSchemaStorageManager.java | 59 ++-
.../io/storage/HoodieAvroFileReaderFactory.java | 20 +-
.../io/storage/HoodieAvroFileWriterFactory.java | 10 +-
.../hudi/io/storage/HoodieAvroHFileWriter.java | 3 +-
.../hudi/io/storage/HoodieAvroOrcReader.java | 7 +-
.../hudi/io/storage/HoodieAvroOrcWriter.java | 3 +-
.../hudi/io/storage/HoodieAvroParquetReader.java | 9 +-
.../hudi/io/storage/HoodieAvroParquetWriter.java | 7 +-
.../hudi/io/storage/HoodieBaseParquetWriter.java | 7 +-
.../hudi/io/storage/HoodieFileReaderFactory.java | 26 +-
.../hudi/io/storage/HoodieFileWriterFactory.java | 12 +-
.../io/storage/HoodieHBaseAvroHFileReader.java | 45 +-
.../apache/hudi/io/storage/HoodieHFileUtils.java | 32 +-
.../io/storage/HoodieNativeAvroHFileReader.java | 17 +-
.../hudi/metadata/AbstractHoodieTableMetadata.java | 6 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 91 ++--
.../metadata/FileSystemBackedTableMetadata.java | 114 +++--
.../hudi/metadata/HoodieBackedTableMetadata.java | 6 +-
.../metadata/HoodieMetadataFileSystemView.java | 14 +-
.../metadata/HoodieMetadataLogRecordReader.java | 6 +-
.../hudi/metadata/HoodieMetadataMetrics.java | 2 +-
.../hudi/metadata/HoodieMetadataPayload.java | 32 +-
.../apache/hudi/metadata/HoodieTableMetadata.java | 13 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 73 +--
.../apache/hudi/storage/HoodieStorageUtils.java | 55 ++
.../avro/HoodieAvroParquetReaderBuilder.java | 6 +-
.../hudi/common/bootstrap/TestBootstrapIndex.java | 4 +-
.../org/apache/hudi/common/fs/TestFSUtils.java | 147 +++---
.../fs/TestFSUtilsWithRetryWrapperEnable.java | 57 ++-
.../common/fs/TestHoodieWrapperFileSystem.java | 16 +-
.../hudi/common/fs/inline/InLineFSUtilsTest.java | 8 +-
.../common/fs/inline/TestInLineFileSystem.java | 73 +--
.../TestInLineFileSystemHFileInLiningBase.java | 3 +-
.../hudi/common/fs/inline/TestParquetInLining.java | 3 +-
.../common/functional/TestHoodieLogFormat.java | 557 ++++++++++++---------
.../TestHoodieLogFormatAppendFailure.java | 29 +-
.../apache/hudi/common/model/TestFileSlice.java | 7 +-
.../hudi/common/model/TestHoodieBaseFile.java | 52 +-
.../hudi/common/model/TestHoodieFileGroup.java | 8 +-
.../hudi/common/model/TestHoodieLogFile.java | 36 +-
.../common/model/TestHoodiePartitionMetadata.java | 38 +-
.../hudi/common/model/TestHoodieWriteStat.java | 12 +-
.../hudi/common/table/TestHoodieTableConfig.java | 100 ++--
.../hudi/common/table/TestTableSchemaResolver.java | 17 +-
.../table/timeline/TestHoodieActiveTimeline.java | 30 +-
.../table/view/TestHoodieTableFileSystemView.java | 337 ++++++++-----
.../table/view/TestIncrementalFSViewSync.java | 44 +-
.../apache/hudi/common/testutils/Assertions.java | 10 +
.../hudi/common/testutils/CompactionTestUtils.java | 13 +-
.../hudi/common/testutils/FileCreateUtils.java | 186 ++++---
.../hudi/common/testutils/FileSystemTestUtils.java | 25 +-
.../common/testutils/HoodieTestDataGenerator.java | 14 +-
.../hudi/common/testutils/HoodieTestTable.java | 69 ++-
.../hudi/common/testutils/HoodieTestUtils.java | 36 +-
.../reader/HoodieFileGroupReaderTestHarness.java | 2 +-
.../testutils/reader/HoodieFileSliceTestUtils.java | 29 +-
.../testutils/reader/HoodieTestReaderContext.java | 14 +-
.../apache/hudi/common/util/TestCommitUtils.java | 3 +-
.../hudi/common/util/TestCompactionUtils.java | 46 +-
.../util/TestDFSPropertiesConfiguration.java | 20 +-
.../apache/hudi/common/util/TestMarkerUtils.java | 43 +-
.../apache/hudi/common/util/TestParquetUtils.java | 13 +-
.../hudi/common/util/TestTablePathUtils.java | 95 ++--
.../storage/TestHoodieAvroFileReaderFactory.java | 9 +-
.../io/storage/TestHoodieBaseParquetWriter.java | 16 +-
.../storage/TestHoodieHBaseHFileReaderWriter.java | 13 +-
.../storage/TestHoodieHFileReaderWriterBase.java | 14 +-
.../hudi/io/storage/TestHoodieOrcReaderWriter.java | 9 +-
.../io/storage/TestHoodieReaderWriterBase.java | 4 +-
.../TestFileSystemBackedTableMetadata.java | 85 ++--
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 13 +-
.../examples/quickstart/TestQuickstartData.java | 23 +-
.../hudi/sink/bootstrap/BootstrapOperator.java | 10 +-
.../hudi/sink/clustering/ClusteringOperator.java | 10 +-
.../sink/clustering/FlinkClusteringConfig.java | 4 +-
.../hudi/sink/compact/FlinkCompactionConfig.java | 4 +-
.../sink/partitioner/profile/WriteProfiles.java | 42 +-
.../java/org/apache/hudi/source/FileIndex.java | 41 +-
.../apache/hudi/source/IncrementalInputSplits.java | 37 +-
.../org/apache/hudi/table/HoodieTableSource.java | 23 +-
.../apache/hudi/table/format/FilePathUtils.java | 4 +
.../org/apache/hudi/table/format/FormatUtils.java | 16 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 16 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 34 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 11 +-
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 21 +-
.../bucket/ITTestConsistentBucketStreamWrite.java | 9 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 13 +-
.../org/apache/hudi/sink/utils/TestWriteBase.java | 20 +-
.../java/org/apache/hudi/source/TestFileIndex.java | 46 +-
.../hudi/source/TestIncrementalInputSplits.java | 17 +-
.../apache/hudi/table/TestHoodieTableSource.java | 63 ++-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 17 +-
.../test/java/org/apache/hudi/utils/TestData.java | 12 +-
.../test/java/org/apache/hudi/utils/TestUtils.java | 3 +-
.../apache/hudi/hadoop/fs/ConsistencyGuard.java | 14 +-
.../org/apache/hudi/hadoop/fs/HadoopFSUtils.java | 27 +
.../hudi/hadoop/fs/HoodieWrapperFileSystem.java | 113 +----
.../hudi/hadoop/fs/NoOpConsistencyGuard.java | 8 +-
.../hadoop/fs/SizeAwareFSDataOutputStream.java | 3 +-
.../hudi/hadoop/fs/inline/InLineFSUtils.java | 35 +-
.../hudi/hadoop/fs/inline/InLineFileSystem.java | 7 +-
.../storage/hadoop/TestHoodieHadoopStorage.java | 2 +-
.../hudi/hadoop/HiveHoodieTableFileIndex.java | 11 +-
.../hadoop/HoodieCopyOnWriteTableInputFormat.java | 3 +-
.../hudi/hadoop/HoodieHFileRecordReader.java | 4 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 18 +-
.../apache/hudi/hadoop/SchemaEvolutionContext.java | 11 +-
.../realtime/HoodieMergeOnReadSnapshotReader.java | 4 +-
.../HoodieMergeOnReadTableInputFormat.java | 42 +-
.../realtime/RealtimeCompactedRecordReader.java | 4 +-
.../apache/hudi/hadoop/realtime/RealtimeSplit.java | 4 +-
.../realtime/RealtimeUnmergedRecordReader.java | 5 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 54 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 4 +-
.../hudi/hadoop/TestHoodieROTablePathFilter.java | 4 +-
.../hive/TestHoodieCombineHiveInputFormat.java | 17 +-
.../TestHoodieMergeOnReadSnapshotReader.java | 20 +-
.../TestHoodieMergeOnReadTableInputFormat.java | 7 +-
.../realtime/TestHoodieRealtimeFileSplit.java | 3 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 54 +-
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 83 ++-
.../testsuite/HoodieDeltaStreamerWrapper.java | 2 +-
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 10 +-
.../SparkDataSourceContinuousIngestTool.java | 3 +-
.../integ/testsuite/dag/nodes/RollbackNode.java | 15 +-
.../helpers/DFSTestSuitePathSelector.java | 41 +-
.../reader/DFSHoodieDatasetInputReader.java | 14 +-
.../testsuite/writer/AvroFileDeltaInputWriter.java | 3 +-
.../testsuite/job/TestHoodieTestSuiteJob.java | 8 +-
hudi-io/pom.xml | 217 ++++----
.../org/apache/hudi/common/util/FileIOUtils.java | 79 ++-
.../hudi/io/storage/TestHoodieStorageBase.java | 28 +-
.../timeline/HoodieMetaserverBasedTimeline.java | 16 +-
.../main/java/org/apache/hudi/DataSourceUtils.java | 15 +-
.../org/apache/hudi/BaseFileOnlyRelation.scala | 13 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 25 +-
.../scala/org/apache/hudi/FileSystemRelation.scala | 10 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 40 +-
.../org/apache/hudi/HoodieBootstrapMORRDD.scala | 13 +-
.../apache/hudi/HoodieBootstrapMORRelation.scala | 5 +-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 29 +-
.../org/apache/hudi/HoodieDataSourceHelper.scala | 16 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 55 +-
.../hudi/HoodieHadoopFsRelationFactory.scala | 21 +-
.../apache/hudi/HoodieIncrementalFileIndex.scala | 24 +-
.../org/apache/hudi/IncrementalRelation.scala | 19 +-
.../src/main/scala/org/apache/hudi/Iterators.scala | 64 ++-
.../hudi/MergeOnReadIncrementalRelation.scala | 18 +-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 11 +-
.../org/apache/hudi/RecordLevelIndexSupport.scala | 10 +-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 44 +-
.../org/apache/hudi/cdc/CDCFileGroupIterator.scala | 43 +-
.../scala/org/apache/hudi/cdc/HoodieCDCRDD.scala | 34 +-
.../datasources/HoodieInMemoryFileIndex.scala | 10 +-
.../datasources/HoodieMultipleBaseFileFormat.scala | 18 +-
...odieFileGroupReaderBasedParquetFileFormat.scala | 19 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 20 +-
.../AlterHoodieTableAddPartitionCommand.scala | 15 +-
.../sql/hudi/command/DropHoodieTableCommand.scala | 11 +-
.../hudi/command/RepairHoodieTableCommand.scala | 10 +-
.../hudi/command/TruncateHoodieTableCommand.scala | 13 +-
.../sql/hudi/streaming/HoodieStreamSource.scala | 17 +-
.../datasources/TestHoodieInMemoryFileIndex.scala | 8 +-
.../org/apache/hudi/HoodieDataSourceHelpers.java | 47 +-
.../apache/hudi/cli/HDFSParquetImporterUtils.java | 5 +-
.../org/apache/spark/sql/hudi/DedupeSparkJob.scala | 50 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 28 +-
.../procedures/CreateMetadataTableProcedure.scala | 11 +-
.../procedures/ExportInstantsProcedure.scala | 19 +-
.../procedures/InitMetadataTableProcedure.scala | 7 +-
.../RepairAddpartitionmetaProcedure.scala | 15 +-
.../RepairCorruptedCleanFilesProcedure.scala | 4 +-
.../procedures/RepairDeduplicateProcedure.scala | 7 +-
.../RepairMigratePartitionMetaProcedure.scala | 24 +-
.../RepairOverwriteHoodiePropsProcedure.scala | 6 +-
.../command/procedures/RunBootstrapProcedure.scala | 9 +-
.../ShowColumnStatsOverlapProcedure.scala | 18 +-
.../procedures/ShowFileSystemViewProcedure.scala | 11 +-
.../ShowHoodieLogFileMetadataProcedure.scala | 21 +-
.../ShowHoodieLogFileRecordsProcedure.scala | 17 +-
.../procedures/ShowInvalidParquetProcedure.scala | 4 +-
.../ShowMetadataTableColumnStatsProcedure.scala | 14 +-
.../ShowMetadataTableFilesProcedure.scala | 13 +-
.../procedures/StatsFileSizeProcedure.scala | 12 +-
.../ValidateMetadataTableFilesProcedure.scala | 54 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 11 +-
.../TestBulkInsertInternalPartitionerForRows.java | 2 +-
.../org/apache/hudi/functional/TestBootstrap.java | 32 +-
.../TestHoodieSparkMergeOnReadTableClustering.java | 15 +-
.../apache/hudi/functional/TestOrcBootstrap.java | 30 +-
.../TestSparkConsistentBucketClustering.java | 14 +-
.../functional/TestSparkSortAndSizeClustering.java | 3 +-
.../row/TestHoodieInternalRowParquetWriter.java | 9 +-
.../io/storage/row/TestHoodieRowCreateHandle.java | 3 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 16 +-
.../TestSpark35RecordPositionMetadataColumn.scala | 13 +-
.../hudi/functional/ColumnStatIndexTestBase.scala | 13 +-
.../hudi/functional/RecordLevelIndexTestBase.scala | 19 +-
.../TestAutoGenerationOfRecordKeys.scala | 29 +-
.../hudi/functional/TestBasicSchemaEvolution.scala | 15 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 63 +--
.../hudi/functional/TestColumnStatsIndex.scala | 9 +-
.../functional/TestColumnStatsIndexWithSQL.scala | 3 +-
.../apache/hudi/functional/TestEmptyCommit.scala | 9 +-
.../hudi/functional/TestHoodieActiveTimeline.scala | 21 +-
.../TestHoodieMultipleBaseFileFormat.scala | 7 +-
.../TestIncrementalReadByStateTransitionTime.scala | 5 +-
.../TestIncrementalReadWithFullTableScan.scala | 9 +-
.../hudi/functional/TestLayoutOptimization.scala | 9 +-
.../apache/hudi/functional/TestMORDataSource.scala | 44 +-
.../TestMORDataSourceWithBucketIndex.scala | 25 +-
.../hudi/functional/TestMetadataRecordIndex.scala | 11 +-
.../TestMetadataTableWithSparkDataSource.scala | 36 +-
.../hudi/functional/TestMetricsReporter.scala | 12 +-
.../functional/TestPartialUpdateAvroPayload.scala | 13 +-
.../functional/TestSixToFiveDowngradeHandler.scala | 7 +-
.../TestSparkDataSourceDAGExecution.scala | 13 +-
.../hudi/functional/TestStructuredStreaming.scala | 72 +--
.../hudi/functional/TestTimeTravelQuery.scala | 14 +-
.../hudi/functional/cdc/HoodieCDCTestBase.scala | 16 +-
.../scala/org/apache/hudi/util/TestPathUtils.scala | 43 +-
.../apache/spark/sql/hudi/common/TestSqlConf.scala | 7 +-
.../hudi/dml/TestPartialUpdateForMergeInto.scala | 2 +-
.../procedure/TestHdfsParquetImportProcedure.scala | 21 +-
.../sql/hudi/procedure/TestRepairsProcedure.scala | 46 +-
.../TestUpgradeOrDowngradeProcedure.scala | 20 +-
.../apache/spark/sql/adapter/Spark2Adapter.scala | 9 +-
.../HoodieSpark2PartitionedFileUtils.scala | 10 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 2 +-
.../TestHoodieDataSourceInternalWriter.java | 19 +-
.../spark/sql/adapter/BaseSpark3Adapter.scala | 14 +-
.../HoodieSpark30PartitionedFileUtils.scala | 10 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 2 +-
.../TestHoodieDataSourceInternalBatchWrite.java | 18 +-
.../HoodieSpark31PartitionedFileUtils.scala | 10 +-
.../HoodieSpark32PartitionedFileUtils.scala | 10 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 2 +-
.../TestHoodieDataSourceInternalBatchWrite.java | 18 +-
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 17 +-
.../HoodieSpark33PartitionedFileUtils.scala | 10 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 2 +-
.../TestHoodieDataSourceInternalBatchWrite.java | 30 +-
.../HoodieSpark34PartitionedFileUtils.scala | 12 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 2 +-
.../TestHoodieDataSourceInternalBatchWrite.java | 18 +-
.../HoodieSpark35PartitionedFileUtils.scala | 11 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 2 +-
.../TestHoodieDataSourceInternalBatchWrite.java | 18 +-
.../apache/hudi/sync/adb/HoodieAdbJdbcClient.java | 15 +-
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 5 +-
.../hudi/hive/ddl/QueryBasedDDLExecutor.java | 5 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 2 +-
.../apache/hudi/hive/testutils/HiveTestUtil.java | 36 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 10 +-
.../hudi/sync/common/util/ManifestFileWriter.java | 15 +-
.../sync/common/util/TestManifestFileWriter.java | 15 +-
.../hudi/timeline/service/RequestHandler.java | 14 +-
.../hudi/timeline/service/TimelineService.java | 20 +-
.../timeline/service/handlers/BaseFileHandler.java | 6 +-
.../service/handlers/FileSliceHandler.java | 6 +-
.../hudi/timeline/service/handlers/Handler.java | 8 +-
.../service/handlers/InstantStateHandler.java | 21 +-
.../timeline/service/handlers/MarkerHandler.java | 12 +-
.../timeline/service/handlers/TimelineHandler.java | 6 +-
.../AsyncTimelineServerBasedDetectionStrategy.java | 6 +-
.../MarkerBasedEarlyConflictDetectionRunnable.java | 25 +-
.../service/handlers/marker/MarkerDirState.java | 33 +-
.../TestRemoteHoodieTableFileSystemView.java | 12 +-
...tMarkerBasedEarlyConflictDetectionRunnable.java | 17 +-
.../apache/hudi/utilities/HDFSParquetImporter.java | 3 +-
.../hudi/utilities/HoodieDataTableUtils.java | 13 +-
.../hudi/utilities/HoodieDataTableValidator.java | 18 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 6 +-
.../org/apache/hudi/utilities/HoodieIndexer.java | 4 +-
.../utilities/HoodieMetadataTableValidator.java | 34 +-
.../apache/hudi/utilities/HoodieRepairTool.java | 19 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 11 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 11 +-
.../org/apache/hudi/utilities/TableSizeStats.java | 3 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 24 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 4 +-
.../deltastreamer/HoodieDeltaStreamer.java | 5 +-
.../multitable/HoodieMultiTableServicesMain.java | 3 +-
.../hudi/utilities/perf/TimelineServerPerf.java | 33 +-
.../utilities/sources/helpers/DFSPathSelector.java | 55 +-
.../sources/helpers/DatePartitionPathSelector.java | 44 +-
.../utilities/streamer/BaseErrorTableWriter.java | 5 +-
.../hudi/utilities/streamer/ErrorTableUtils.java | 28 +-
.../streamer/HoodieMultiTableStreamer.java | 5 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 61 ++-
.../apache/hudi/utilities/streamer/StreamSync.java | 65 ++-
.../apache/hudi/utilities/TestHoodieIndexer.java | 2 +-
.../TestHoodieMetadataTableValidator.java | 8 +-
.../hudi/utilities/TestHoodieRepairTool.java | 60 ++-
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 69 +--
.../deltastreamer/TestHoodieDeltaStreamer.java | 54 +-
...TestHoodieDeltaStreamerSchemaEvolutionBase.java | 7 +-
...estHoodieDeltaStreamerSchemaEvolutionQuick.java | 24 +-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 102 ++--
.../TestHoodieMultiTableDeltaStreamer.java | 3 +-
.../functional/TestHDFSParquetImporter.java | 62 ++-
.../functional/TestHoodieSnapshotCopier.java | 2 +-
.../functional/TestHoodieSnapshotExporter.java | 77 +--
.../offlinejob/TestHoodieClusteringJob.java | 2 +-
.../utilities/sources/TestAvroKafkaSource.java | 47 +-
.../hudi/utilities/sources/TestJsonDFSSource.java | 2 +-
.../utilities/sources/TestJsonKafkaSource.java | 5 +-
.../utilities/sources/TestSqlFileBasedSource.java | 10 +-
.../sources/helpers/TestCloudObjectsSelector.java | 2 +-
.../helpers/TestDFSPathSelectorCommonMethods.java | 17 +-
.../helpers/TestDatePartitionPathSelector.java | 53 +-
.../sources/helpers/TestS3EventsMetaSelector.java | 2 +-
.../streamer/TestStreamSyncUnitTests.java | 6 +-
.../utilities/testutils/UtilitiesTestBase.java | 27 +-
.../transform/TestSqlFileBasedTransformer.java | 6 +-
575 files changed, 8765 insertions(+), 6440 deletions(-)
diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
index 2050e4f60ff..e06db9f2ba4 100644
--- a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
+++ b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
@@ -18,6 +18,7 @@
package org.apache.hudi.aws.sync;
+import org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory;
import org.apache.hudi.aws.sync.util.GluePartitionFilterGenerator;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.table.TableSchemaResolver;
@@ -31,6 +32,9 @@ import org.apache.hudi.sync.common.HoodieSyncClient;
import org.apache.hudi.sync.common.model.FieldSchema;
import org.apache.hudi.sync.common.model.Partition;
+import org.apache.parquet.schema.MessageType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.glue.GlueAsyncClient;
import software.amazon.awssdk.services.glue.GlueAsyncClientBuilder;
@@ -70,10 +74,6 @@ import software.amazon.awssdk.services.glue.model.Table;
import software.amazon.awssdk.services.glue.model.TableInput;
import software.amazon.awssdk.services.glue.model.UpdateTableRequest;
-import org.apache.parquet.schema.MessageType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.net.URI;
import java.net.URISyntaxException;
import java.time.Instant;
@@ -95,12 +95,12 @@ import java.util.stream.Collectors;
import static org.apache.hudi.aws.utils.S3Utils.s3aToS3;
import static org.apache.hudi.common.util.MapUtils.containsAll;
import static org.apache.hudi.common.util.MapUtils.isNullOrEmpty;
+import static org.apache.hudi.config.GlueCatalogSyncClientConfig.ALL_PARTITIONS_READ_PARALLELISM;
import static org.apache.hudi.config.GlueCatalogSyncClientConfig.CHANGED_PARTITIONS_READ_PARALLELISM;
+import static org.apache.hudi.config.GlueCatalogSyncClientConfig.GLUE_METADATA_FILE_LISTING;
import static org.apache.hudi.config.GlueCatalogSyncClientConfig.META_SYNC_PARTITION_INDEX_FIELDS;
import static org.apache.hudi.config.GlueCatalogSyncClientConfig.META_SYNC_PARTITION_INDEX_FIELDS_ENABLE;
import static org.apache.hudi.config.GlueCatalogSyncClientConfig.PARTITION_CHANGE_PARALLELISM;
-import static org.apache.hudi.config.GlueCatalogSyncClientConfig.GLUE_METADATA_FILE_LISTING;
-import static org.apache.hudi.config.GlueCatalogSyncClientConfig.ALL_PARTITIONS_READ_PARALLELISM;
import static org.apache.hudi.config.HoodieAWSConfig.AWS_GLUE_ENDPOINT;
import static org.apache.hudi.config.HoodieAWSConfig.AWS_GLUE_REGION;
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_CREATE_MANAGED_TABLE;
@@ -110,7 +110,6 @@ import static org.apache.hudi.hive.util.HiveSchemaUtil.parquetSchemaToMapSchema;
import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME;
import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS;
import static org.apache.hudi.sync.common.util.TableUtils.tableId;
-import org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory;
/**
* This class implements all the AWS APIs to enable syncing of a Hudi Table with the
@@ -304,7 +303,7 @@ public class AWSGlueCatalogSyncClient extends HoodieSyncClient {
try {
StorageDescriptor sd = table.storageDescriptor();
List<PartitionInput> partitionInputList = partitionsToAdd.stream().map(partition -> {
- String fullPartitionPath = FSUtils.getPartitionPath(s3aToS3(getBasePath()), partition).toString();
+ String fullPartitionPath = FSUtils.getPartitionPathInHadoopPath(s3aToS3(getBasePath()), partition).toString();
List<String> partitionValues = partitionValueExtractor.extractPartitionValuesInPath(partition);
StorageDescriptor partitionSD = sd.copy(copySd -> copySd.location(fullPartitionPath));
return PartitionInput.builder().values(partitionValues).storageDescriptor(partitionSD).build();
@@ -348,7 +347,7 @@ public class AWSGlueCatalogSyncClient extends HoodieSyncClient {
try {
StorageDescriptor sd = table.storageDescriptor();
List<BatchUpdatePartitionRequestEntry> updatePartitionEntries = changedPartitions.stream().map(partition -> {
- String fullPartitionPath = FSUtils.getPartitionPath(s3aToS3(getBasePath()), partition).toString();
+ String fullPartitionPath = FSUtils.getPartitionPathInHadoopPath(s3aToS3(getBasePath()), partition).toString();
List<String> partitionValues = partitionValueExtractor.extractPartitionValuesInPath(partition);
StorageDescriptor partitionSD = sd.copy(copySd -> copySd.location(fullPartitionPath));
PartitionInput partitionInput = PartitionInput.builder().values(partitionValues).storageDescriptor(partitionSD).build();
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java
index 46a1b715ca3..9cd5b69e03f 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java
@@ -26,6 +26,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -40,7 +42,7 @@ public class HoodieCLI {
public static Configuration conf;
public static ConsistencyGuardConfig consistencyGuardConfig = ConsistencyGuardConfig.newBuilder().build();
public static HoodieTimeGeneratorConfig timeGeneratorConfig;
- public static FileSystem fs;
+ public static HoodieStorage storage;
public static CLIState state = CLIState.INIT;
public static String basePath;
protected static HoodieTableMetaClient tableMetadata;
@@ -85,8 +87,10 @@ public class HoodieCLI {
}
public static void initFS(boolean force) throws IOException {
- if (fs == null || force) {
- fs = (tableMetadata != null) ? tableMetadata.getFs() : FileSystem.get(conf);
+ if (storage == null || force) {
+ storage = (tableMetadata != null)
+ ? tableMetadata.getStorage()
+ : HoodieStorageUtils.getStorage(FileSystem.get(conf));
}
}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java
index 5c57c8f5288..921d12fb663 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java
@@ -37,13 +37,13 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.avro.specific.SpecificData;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
import org.apache.spark.launcher.SparkLauncher;
import org.apache.spark.util.Utils;
import org.slf4j.Logger;
@@ -106,16 +106,18 @@ public class ArchivedCommitsCommand {
throws IOException {
System.out.println("===============> Showing only " + limit + " archived commits <===============");
String basePath = HoodieCLI.getTableMetaClient().getBasePath();
- Path archivePath = new Path(HoodieCLI.getTableMetaClient().getArchivePath() + "/.commits_.archive*");
+ StoragePath archivePath = new StoragePath(
+ HoodieCLI.getTableMetaClient().getArchivePath() + "/.commits_.archive*");
if (folder != null && !folder.isEmpty()) {
- archivePath = new Path(basePath + "/.hoodie/" + folder);
+ archivePath = new StoragePath(basePath + "/.hoodie/" + folder);
}
- FileStatus[] fsStatuses = HadoopFSUtils.getFs(basePath, HoodieCLI.conf).globStatus(archivePath);
+ List<StoragePathInfo> pathInfoList =
+ HoodieStorageUtils.getStorage(basePath, HoodieCLI.conf).globEntries(archivePath);
List<Comparable[]> allStats = new ArrayList<>();
- for (FileStatus fs : fsStatuses) {
+ for (StoragePathInfo pathInfo : pathInfoList) {
// read the archived file
- try (Reader reader = HoodieLogFormat.newReader(HadoopFSUtils.getFs(basePath, HoodieCLI.conf),
- new HoodieLogFile(fs.getPath()), HoodieArchivedMetaEntry.getClassSchema())) {
+ try (Reader reader = HoodieLogFormat.newReader(HoodieStorageUtils.getStorage(basePath, HoodieCLI.conf),
+ new HoodieLogFile(pathInfo.getPath()), HoodieArchivedMetaEntry.getClassSchema())) {
List<IndexedRecord> readRecords = new ArrayList<>();
// read the avro blocks
while (reader.hasNext()) {
@@ -181,14 +183,15 @@ public class ArchivedCommitsCommand {
System.out.println("===============> Showing only " + limit + " archived commits <===============");
HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
String basePath = metaClient.getBasePath();
- Path archivePath = new Path(metaClient.getArchivePath() + "/.commits_.archive*");
- FileStatus[] fsStatuses =
- HadoopFSUtils.getFs(basePath, HoodieCLI.conf).globStatus(archivePath);
+ StoragePath archivePath =
+ new StoragePath(metaClient.getArchivePath() + "/.commits_.archive*");
+ List<StoragePathInfo> pathInfoList =
+ HoodieStorageUtils.getStorage(basePath, HoodieCLI.conf).globEntries(archivePath);
List<Comparable[]> allCommits = new ArrayList<>();
- for (FileStatus fs : fsStatuses) {
+ for (StoragePathInfo pathInfo : pathInfoList) {
// read the archived file
- try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(HadoopFSUtils.getFs(basePath, HoodieCLI.conf),
- new HoodieLogFile(fs.getPath()), HoodieArchivedMetaEntry.getClassSchema())) {
+ try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(HoodieStorageUtils.getStorage(basePath, HoodieCLI.conf),
+ new HoodieLogFile(pathInfo.getPath()), HoodieArchivedMetaEntry.getClassSchema())) {
List<IndexedRecord> readRecords = new ArrayList<>();
// read the avro blocks
while (reader.hasNext()) {
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java
index b9ab2f47ed2..135bd48c036 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java
@@ -42,11 +42,11 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.action.compact.OperationResult;
import org.apache.hudi.utilities.UtilHelpers;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.spark.launcher.SparkLauncher;
import org.apache.spark.util.Utils;
import org.slf4j.Logger;
@@ -432,9 +432,9 @@ public class CompactionCommand {
return TMP_DIR + UUID.randomUUID().toString() + ".ser";
}
- private <T> T deSerializeOperationResult(String inputP, FileSystem fs) throws Exception {
- Path inputPath = new Path(inputP);
- InputStream inputStream = fs.open(inputPath);
+ private <T> T deSerializeOperationResult(StoragePath inputPath,
+ HoodieStorage storage) throws Exception {
+ InputStream inputStream = storage.open(inputPath);
ObjectInputStream in = new ObjectInputStream(inputStream);
try {
T result = (T) in.readObject();
@@ -463,7 +463,7 @@ public class CompactionCommand {
HoodieCLI.initFS(initialized);
String outputPathStr = getTmpSerializerFile();
- Path outputPath = new Path(outputPathStr);
+ StoragePath outputPath = new StoragePath(outputPathStr);
String output;
try {
String sparkPropertiesPath = Utils
@@ -477,7 +477,7 @@ public class CompactionCommand {
if (exitCode != 0) {
return "Failed to validate compaction for " + compactionInstant;
}
- List<ValidationOpResult> res = deSerializeOperationResult(outputPathStr, HoodieCLI.fs);
+ List<ValidationOpResult> res = deSerializeOperationResult(outputPath, HoodieCLI.storage);
boolean valid = res.stream().map(OperationResult::isSuccess).reduce(Boolean::logicalAnd).orElse(true);
String message = "\n\n\t COMPACTION PLAN " + (valid ? "VALID" : "INVALID") + "\n\n";
List<Comparable[]> rows = new ArrayList<>();
@@ -502,8 +502,8 @@ public class CompactionCommand {
headerOnly, rows);
} finally {
// Delete tmp file used to serialize result
- if (HoodieCLI.fs.exists(outputPath)) {
- HoodieCLI.fs.delete(outputPath, false);
+ if (HoodieCLI.storage.exists(outputPath)) {
+ HoodieCLI.storage.deleteFile(outputPath);
}
}
return output;
@@ -528,7 +528,7 @@ public class CompactionCommand {
HoodieCLI.initFS(initialized);
String outputPathStr = getTmpSerializerFile();
- Path outputPath = new Path(outputPathStr);
+ StoragePath outputPath = new StoragePath(outputPathStr);
String output;
try {
String sparkPropertiesPath = Utils
@@ -543,13 +543,13 @@ public class CompactionCommand {
if (exitCode != 0) {
return "Failed to unschedule compaction for " + compactionInstant;
}
- List<RenameOpResult> res = deSerializeOperationResult(outputPathStr, HoodieCLI.fs);
+ List<RenameOpResult> res = deSerializeOperationResult(outputPath, HoodieCLI.storage);
output =
getRenamesToBePrinted(res, limit, sortByField, descending, headerOnly, "unschedule pending compaction");
} finally {
// Delete tmp file used to serialize result
- if (HoodieCLI.fs.exists(outputPath)) {
- HoodieCLI.fs.delete(outputPath, false);
+ if (HoodieCLI.storage.exists(outputPath)) {
+ HoodieCLI.storage.deleteFile(outputPath);
}
}
return output;
@@ -573,7 +573,7 @@ public class CompactionCommand {
HoodieCLI.initFS(initialized);
String outputPathStr = getTmpSerializerFile();
- Path outputPath = new Path(outputPathStr);
+ StoragePath outputPath = new StoragePath(outputPathStr);
String output;
try {
String sparkPropertiesPath = Utils
@@ -588,13 +588,13 @@ public class CompactionCommand {
if (exitCode != 0) {
return "Failed to unschedule compaction for file " + fileId;
}
- List<RenameOpResult> res = deSerializeOperationResult(outputPathStr, HoodieCLI.fs);
+ List<RenameOpResult> res = deSerializeOperationResult(outputPath, HoodieCLI.storage);
output = getRenamesToBePrinted(res, limit, sortByField, descending, headerOnly,
"unschedule file from pending compaction");
} finally {
// Delete tmp file used to serialize result
- if (HoodieCLI.fs.exists(outputPath)) {
- HoodieCLI.fs.delete(outputPath, false);
+ if (HoodieCLI.storage.exists(outputPath)) {
+ HoodieCLI.storage.deleteFile(outputPath);
}
}
return output;
@@ -619,7 +619,7 @@ public class CompactionCommand {
HoodieCLI.initFS(initialized);
String outputPathStr = getTmpSerializerFile();
- Path outputPath = new Path(outputPathStr);
+ StoragePath outputPath = new StoragePath(outputPathStr);
String output;
try {
String sparkPropertiesPath = Utils
@@ -633,12 +633,12 @@ public class CompactionCommand {
if (exitCode != 0) {
return "Failed to unschedule compaction for " + compactionInstant;
}
- List<RenameOpResult> res = deSerializeOperationResult(outputPathStr, HoodieCLI.fs);
+ List<RenameOpResult> res = deSerializeOperationResult(outputPath, HoodieCLI.storage);
output = getRenamesToBePrinted(res, limit, sortByField, descending, headerOnly, "repair compaction");
} finally {
// Delete tmp file used to serialize result
- if (HoodieCLI.fs.exists(outputPath)) {
- HoodieCLI.fs.delete(outputPath, false);
+ if (HoodieCLI.storage.exists(outputPath)) {
+ HoodieCLI.storage.deleteFile(outputPath);
}
}
return output;
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java
index eda0d0de219..b0152c8a192 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java
@@ -37,15 +37,14 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.avro.specific.SpecificData;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.shell.standard.ShellComponent;
@@ -84,7 +83,7 @@ public class ExportCommand {
throws Exception {
final String basePath = HoodieCLI.getTableMetaClient().getBasePath();
- final Path archivePath = new Path(HoodieCLI.getTableMetaClient().getArchivePath());
+ final StoragePath archivePath = new StoragePath(HoodieCLI.getTableMetaClient().getArchivePath());
final Set<String> actionSet = new HashSet<String>(Arrays.asList(filter.split(",")));
int numExports = limit == -1 ? Integer.MAX_VALUE : limit;
int numCopied = 0;
@@ -99,18 +98,21 @@ public class ExportCommand {
List<HoodieInstant> nonArchivedInstants = timeline.getInstants();
// Archived instants are in the commit archive files
- FileStatus[] statuses = HadoopFSUtils.getFs(basePath, HoodieCLI.conf).globStatus(archivePath);
- List<FileStatus> archivedStatuses = Arrays.stream(statuses).sorted((f1, f2) -> (int) (f1.getModificationTime() - f2.getModificationTime())).collect(Collectors.toList());
+ List<StoragePathInfo> pathInfoList =
+ HoodieStorageUtils.getStorage(basePath, HoodieCLI.conf).globEntries(archivePath);
+ List<StoragePathInfo> archivedPathInfoList = pathInfoList.stream()
+ .sorted((f1, f2) -> (int) (f1.getModificationTime() - f2.getModificationTime()))
+ .collect(Collectors.toList());
if (descending) {
Collections.reverse(nonArchivedInstants);
numCopied = copyNonArchivedInstants(nonArchivedInstants, numExports, localFolder);
if (numCopied < numExports) {
- Collections.reverse(archivedStatuses);
- numCopied += copyArchivedInstants(archivedStatuses, actionSet, numExports - numCopied, localFolder);
+ Collections.reverse(archivedPathInfoList);
+ numCopied += copyArchivedInstants(archivedPathInfoList, actionSet, numExports - numCopied, localFolder);
}
} else {
- numCopied = copyArchivedInstants(archivedStatuses, actionSet, numExports, localFolder);
+ numCopied = copyArchivedInstants(archivedPathInfoList, actionSet, numExports, localFolder);
if (numCopied < numExports) {
numCopied += copyNonArchivedInstants(nonArchivedInstants, numExports - numCopied, localFolder);
}
@@ -119,13 +121,17 @@ public class ExportCommand {
return "Exported " + numCopied + " Instants to " + localFolder;
}
- private int copyArchivedInstants(List<FileStatus> statuses, Set<String> actionSet, int limit, String localFolder) throws Exception {
+ private int copyArchivedInstants(List<StoragePathInfo> pathInfoList,
+ Set<String> actionSet,
+ int limit,
+ String localFolder) throws Exception {
int copyCount = 0;
- FileSystem fileSystem = HadoopFSUtils.getFs(HoodieCLI.getTableMetaClient().getBasePath(), HoodieCLI.conf);
+ HoodieStorage storage = HoodieStorageUtils.getStorage(
+ HoodieCLI.getTableMetaClient().getBasePath(), HoodieCLI.conf);
- for (FileStatus fs : statuses) {
+ for (StoragePathInfo pathInfo : pathInfoList) {
// read the archived file
- try (Reader reader = HoodieLogFormat.newReader(fileSystem, new HoodieLogFile(fs.getPath()), HoodieArchivedMetaEntry.getClassSchema())) {
+ try (Reader reader = HoodieLogFormat.newReader(storage, new HoodieLogFile(pathInfo.getPath()), HoodieArchivedMetaEntry.getClassSchema())) {
// read the avro blocks
while (reader.hasNext() && copyCount++ < limit) {
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/FileSystemViewCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/FileSystemViewCommand.java
index 3cd7b52dd54..e4a2bf4f6fb 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/FileSystemViewCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/FileSystemViewCommand.java
@@ -18,9 +18,6 @@
package org.apache.hudi.cli.commands;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.cli.HoodieCLI;
import org.apache.hudi.cli.HoodiePrintHelper;
import org.apache.hudi.cli.HoodieTableHeaderFields;
@@ -35,6 +32,10 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.util.NumericUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorage;
+
import org.springframework.shell.standard.ShellComponent;
import org.springframework.shell.standard.ShellMethod;
import org.springframework.shell.standard.ShellOption;
@@ -236,11 +237,12 @@ public class FileSystemViewCommand {
private HoodieTableFileSystemView buildFileSystemView(String globRegex, String maxInstant, boolean basefileOnly,
boolean includeMaxInstant, boolean includeInflight, boolean excludeCompaction) throws IOException {
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
- HoodieTableMetaClient metaClient =
- HoodieTableMetaClient.builder().setConf(client.getHadoopConf()).setBasePath(client.getBasePath()).setLoadActiveTimelineOnLoad(true).build();
- FileSystem fs = HoodieCLI.fs;
+ HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
+ .setConf(client.getHadoopConf())
+ .setBasePath(client.getBasePath()).setLoadActiveTimelineOnLoad(true).build();
+ HoodieStorage storage = HoodieCLI.storage;
String globPath = String.format("%s/%s/*", client.getBasePath(), globRegex);
- List<FileStatus> statuses = FSUtils.getGlobStatusExcludingMetaFolder(fs, new Path(globPath));
+ List<StoragePathInfo> pathInfoList = FSUtils.getGlobStatusExcludingMetaFolder(storage, new StoragePath(globPath));
Stream<HoodieInstant> instantsStream;
HoodieTimeline timeline;
@@ -270,6 +272,6 @@ public class FileSystemViewCommand {
HoodieTimeline filteredTimeline = new HoodieDefaultTimeline(instantsStream,
(Function<HoodieInstant, Option<byte[]>> & Serializable) metaClient.getActiveTimeline()::getInstantDetails);
- return new HoodieTableFileSystemView(metaClient, filteredTimeline, statuses.toArray(new FileStatus[0]));
+ return new HoodieTableFileSystemView(metaClient, filteredTimeline, pathInfoList);
}
}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java
index fc1a4b4ec42..367dc2302ee 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java
@@ -43,14 +43,12 @@ import org.apache.hudi.common.table.log.block.HoodieLogBlock.HoodieLogBlockType;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
-import org.apache.hudi.hadoop.fs.CachingPath;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.schema.MessageType;
import org.springframework.shell.standard.ShellComponent;
@@ -90,8 +88,9 @@ public class HoodieLogFileCommand {
defaultValue = "false") final boolean headerOnly)
throws IOException {
- FileSystem fs = HoodieCLI.getTableMetaClient().getFs();
- List<String> logFilePaths = FSUtils.getGlobStatusExcludingMetaFolder(fs, new Path(logFilePathPattern)).stream()
+ HoodieStorage storage = HoodieCLI.getTableMetaClient().getStorage();
+ List<String> logFilePaths = FSUtils.getGlobStatusExcludingMetaFolder(
+ storage, new StoragePath(logFilePathPattern)).stream()
.map(status -> status.getPath().toString()).collect(Collectors.toList());
Map<String, List<Tuple3<Tuple2<String, HoodieLogBlockType>, Tuple2<Map<HeaderMetadataType, String>,
Map<HeaderMetadataType, String>>, Integer>>> commitCountAndMetadata =
@@ -101,7 +100,7 @@ public class HoodieLogFileCommand {
String basePath = HoodieCLI.getTableMetaClient().getBasePathV2().toString();
for (String logFilePath : logFilePaths) {
- Path path = new Path(logFilePath);
+ StoragePath path = new StoragePath(logFilePath);
String pathString = path.toString();
String fileName;
if (pathString.contains(basePath)) {
@@ -110,11 +109,10 @@ public class HoodieLogFileCommand {
} else {
fileName = path.getName();
}
- FileStatus[] fsStatus = fs.listStatus(path);
- MessageType schema = TableSchemaResolver.readSchemaFromLogFile(fs, path);
+ MessageType schema = TableSchemaResolver.readSchemaFromLogFile(storage, path);
Schema writerSchema = schema != null
? new AvroSchemaConverter().convert(Objects.requireNonNull(schema)) : null;
- try (Reader reader = HoodieLogFormat.newReader(fs, new HoodieLogFile(fsStatus[0].getPath()), writerSchema)) {
+ try (Reader reader = HoodieLogFormat.newReader(storage, new HoodieLogFile(path), writerSchema)) {
// read the avro blocks
while (reader.hasNext()) {
@@ -205,8 +203,9 @@ public class HoodieLogFileCommand {
System.out.println("===============> Showing only " + limit + " records <===============");
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
- FileSystem fs = client.getFs();
- List<String> logFilePaths = FSUtils.getGlobStatusExcludingMetaFolder(fs, new Path(logFilePathPattern)).stream()
+ HoodieStorage storage = client.getStorage();
+ List<String> logFilePaths = FSUtils.getGlobStatusExcludingMetaFolder(
+ storage, new StoragePath(logFilePathPattern)).stream()
.map(status -> status.getPath().toString()).sorted(Comparator.reverseOrder())
.collect(Collectors.toList());
@@ -218,7 +217,8 @@ public class HoodieLogFileCommand {
Schema readerSchema = null;
// get schema from last log file
for (int i = logFilePaths.size() - 1; i >= 0; i--) {
- MessageType schema = TableSchemaResolver.readSchemaFromLogFile(fs, new Path(logFilePaths.get(i)));
+ MessageType schema = TableSchemaResolver.readSchemaFromLogFile(
+ storage, new StoragePath(logFilePaths.get(i)));
if (schema != null) {
readerSchema = converter.convert(schema);
break;
@@ -231,7 +231,7 @@ public class HoodieLogFileCommand {
System.out.println("===========================> MERGING RECORDS <===================");
HoodieMergedLogRecordScanner scanner =
HoodieMergedLogRecordScanner.newBuilder()
- .withFileSystem(fs)
+ .withStorage(storage)
.withBasePath(client.getBasePath())
.withLogFilePaths(logFilePaths)
.withReaderSchema(readerSchema)
@@ -257,11 +257,12 @@ public class HoodieLogFileCommand {
}
} else {
for (String logFile : logFilePaths) {
- MessageType schema = TableSchemaResolver.readSchemaFromLogFile(client.getFs(), new CachingPath(logFile));
+ MessageType schema = TableSchemaResolver.readSchemaFromLogFile(
+ client.getStorage(), new StoragePath(logFile));
Schema writerSchema = schema != null
? new AvroSchemaConverter().convert(Objects.requireNonNull(schema)) : null;
try (HoodieLogFormat.Reader reader =
- HoodieLogFormat.newReader(fs, new HoodieLogFile(new CachingPath(logFile)), writerSchema)) {
+ HoodieLogFormat.newReader(storage, new HoodieLogFile(new StoragePath(logFile)), writerSchema)) {
// read the avro blocks
while (reader.hasNext()) {
HoodieLogBlock n = reader.next();
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
index 7cebf43db02..b8ccabf150c 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
@@ -37,11 +37,10 @@ import org.apache.hudi.metadata.HoodieTableMetadataUtil;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.metadata.MetadataPartitionType;
import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
-import org.apache.spark.api.java.JavaSparkContext;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.spark.api.java.JavaSparkContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.shell.standard.ShellComponent;
@@ -117,15 +116,15 @@ public class MetadataCommand {
@ShellOption(value = "--sparkMaster", defaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master
) throws Exception {
HoodieCLI.getTableMetaClient();
- Path metadataPath = new Path(getMetadataTableBasePath(HoodieCLI.basePath));
+ StoragePath metadataPath = new StoragePath(getMetadataTableBasePath(HoodieCLI.basePath));
try {
- FileStatus[] statuses = HoodieCLI.fs.listStatus(metadataPath);
- if (statuses.length > 0) {
+ List<StoragePathInfo> pathInfoList = HoodieCLI.storage.listDirectEntries(metadataPath);
+ if (pathInfoList.size() > 0) {
throw new RuntimeException("Metadata directory (" + metadataPath + ") not empty.");
}
} catch (FileNotFoundException e) {
// Metadata directory does not exist yet
- HoodieCLI.fs.mkdirs(metadataPath);
+ HoodieCLI.storage.createDirectory(metadataPath);
}
HoodieTimer timer = HoodieTimer.start();
@@ -164,9 +163,9 @@ public class MetadataCommand {
@ShellOption(value = {"--readonly"}, defaultValue = "false",
help = "Open in read-only mode") final boolean readOnly) throws Exception {
HoodieCLI.getTableMetaClient();
- Path metadataPath = new Path(getMetadataTableBasePath(HoodieCLI.basePath));
+ StoragePath metadataPath = new StoragePath(getMetadataTableBasePath(HoodieCLI.basePath));
try {
- HoodieCLI.fs.listStatus(metadataPath);
+ HoodieCLI.storage.listDirectEntries(metadataPath);
} catch (FileNotFoundException e) {
// Metadata directory does not exist
throw new RuntimeException("Metadata directory (" + metadataPath + ") does not exist.");
@@ -250,24 +249,27 @@ public class MetadataCommand {
return "[ERROR] Metadata Table not enabled/initialized\n\n";
}
- Path partitionPath = new Path(HoodieCLI.basePath);
+ StoragePath partitionPath = new StoragePath(HoodieCLI.basePath);
if (!StringUtils.isNullOrEmpty(partition)) {
- partitionPath = new Path(HoodieCLI.basePath, partition);
+ partitionPath = new StoragePath(HoodieCLI.basePath, partition);
}
HoodieTimer timer = HoodieTimer.start();
- FileStatus[] statuses = metaReader.getAllFilesInPartition(partitionPath);
+ List<StoragePathInfo> pathInfoList = metaReader.getAllFilesInPartition(partitionPath);
LOG.debug("Took " + timer.endTimer() + " ms");
final List<Comparable[]> rows = new ArrayList<>();
- Arrays.stream(statuses).sorted((p1, p2) -> p2.getPath().getName().compareTo(p1.getPath().getName())).forEach(f -> {
- Comparable[] row = new Comparable[1];
- row[0] = f;
- rows.add(row);
- });
+ pathInfoList.stream()
+ .sorted((p1, p2) -> p2.getPath().getName().compareTo(p1.getPath().getName()))
+ .forEach(f -> {
+ Comparable[] row = new Comparable[1];
+ row[0] = f;
+ rows.add(row);
+ });
TableHeader header = new TableHeader().addTableHeaderField("file path");
- return HoodiePrintHelper.print(header, new HashMap<>(), "", false, Integer.MAX_VALUE, false, rows);
+ return HoodiePrintHelper.print(header, new HashMap<>(), "", false, Integer.MAX_VALUE, false,
+ rows);
}
}
@@ -306,26 +308,29 @@ public class MetadataCommand {
final List<Comparable[]> rows = new ArrayList<>();
for (String partition : allPartitions) {
- Map<String, FileStatus> fileStatusMap = new HashMap<>();
- Map<String, FileStatus> metadataFileStatusMap = new HashMap<>();
- FileStatus[] metadataStatuses = metadataReader.getAllFilesInPartition(new Path(HoodieCLI.basePath, partition));
- Arrays.stream(metadataStatuses).forEach(entry -> metadataFileStatusMap.put(entry.getPath().getName(), entry));
- FileStatus[] fsStatuses = fsMetaReader.getAllFilesInPartition(new Path(HoodieCLI.basePath, partition));
- Arrays.stream(fsStatuses).forEach(entry -> fileStatusMap.put(entry.getPath().getName(), entry));
+ Map<String, StoragePathInfo> pathInfoMap = new HashMap<>();
+ Map<String, StoragePathInfo> metadataPathInfoMap = new HashMap<>();
+ List<StoragePathInfo> metadataPathInfoList = metadataReader.getAllFilesInPartition(
+ new StoragePath(HoodieCLI.basePath, partition));
+ metadataPathInfoList.forEach(entry -> metadataPathInfoMap.put(
+ entry.getPath().getName(), entry));
+ List<StoragePathInfo> pathInfoList =
+ fsMetaReader.getAllFilesInPartition(new StoragePath(HoodieCLI.basePath, partition));
+ pathInfoList.forEach(entry -> pathInfoMap.put(entry.getPath().getName(), entry));
Set<String> allFiles = new HashSet<>();
- allFiles.addAll(fileStatusMap.keySet());
- allFiles.addAll(metadataFileStatusMap.keySet());
+ allFiles.addAll(pathInfoMap.keySet());
+ allFiles.addAll(metadataPathInfoMap.keySet());
for (String file : allFiles) {
Comparable[] row = new Comparable[6];
row[0] = partition;
- FileStatus fsFileStatus = fileStatusMap.get(file);
- FileStatus metaFileStatus = metadataFileStatusMap.get(file);
- boolean doesFsFileExists = fsFileStatus != null;
- boolean doesMetadataFileExists = metaFileStatus != null;
- long fsFileLength = doesFsFileExists ? fsFileStatus.getLen() : 0;
- long metadataFileLength = doesMetadataFileExists ? metaFileStatus.getLen() : 0;
+ StoragePathInfo pathInfo = pathInfoMap.get(file);
+ StoragePathInfo metaPathInfo = metadataPathInfoMap.get(file);
+ boolean doesFsFileExists = pathInfo != null;
+ boolean doesMetadataFileExists = metaPathInfo != null;
+ long fsFileLength = doesFsFileExists ? pathInfo.getLength() : 0;
+ long metadataFileLength = doesMetadataFileExists ? metaPathInfo.getLength() : 0;
row[1] = file;
row[2] = doesFsFileExists;
row[3] = doesMetadataFileExists;
@@ -333,37 +338,42 @@ public class MetadataCommand {
row[5] = metadataFileLength;
if (verbose) { // if verbose print all files
rows.add(row);
- } else if ((doesFsFileExists != doesMetadataFileExists) || (fsFileLength != metadataFileLength)) { // if non verbose, print only non matching files
+ } else if ((doesFsFileExists != doesMetadataFileExists)
+ || (fsFileLength != metadataFileLength)) {
+ // if non verbose, print only non matching files
rows.add(row);
}
}
- if (metadataStatuses.length != fsStatuses.length) {
- LOG.error(" FS and metadata files count not matching for " + partition + ". FS files count " + fsStatuses.length + ", metadata base files count "
- + metadataStatuses.length);
+ if (metadataPathInfoList.size() != pathInfoList.size()) {
+ LOG.error(" FS and metadata files count not matching for " + partition
+ + ". FS files count " + pathInfoList.size()
+ + ", metadata base files count " + metadataPathInfoList.size());
}
- for (Map.Entry<String, FileStatus> entry : fileStatusMap.entrySet()) {
- if (!metadataFileStatusMap.containsKey(entry.getKey())) {
+ for (Map.Entry<String, StoragePathInfo> entry : pathInfoMap.entrySet()) {
+ if (!metadataPathInfoMap.containsKey(entry.getKey())) {
LOG.error("FS file not found in metadata " + entry.getKey());
} else {
- if (entry.getValue().getLen() != metadataFileStatusMap.get(entry.getKey()).getLen()) {
+ if (entry.getValue().getLength()
+ != metadataPathInfoMap.get(entry.getKey()).getLength()) {
LOG.error(" FS file size mismatch " + entry.getKey() + ", size equality "
- + (entry.getValue().getLen() == metadataFileStatusMap.get(entry.getKey()).getLen())
- + ". FS size " + entry.getValue().getLen() + ", metadata size "
- + metadataFileStatusMap.get(entry.getKey()).getLen());
+ + (entry.getValue().getLength()
+ == metadataPathInfoMap.get(entry.getKey()).getLength())
+ + ". FS size " + entry.getValue().getLength()
+ + ", metadata size " + metadataPathInfoMap.get(entry.getKey()).getLength());
}
}
}
- for (Map.Entry<String, FileStatus> entry : metadataFileStatusMap.entrySet()) {
- if (!fileStatusMap.containsKey(entry.getKey())) {
+ for (Map.Entry<String, StoragePathInfo> entry : metadataPathInfoMap.entrySet()) {
+ if (!pathInfoMap.containsKey(entry.getKey())) {
LOG.error("Metadata file not found in FS " + entry.getKey());
} else {
- if (entry.getValue().getLen() != fileStatusMap.get(entry.getKey()).getLen()) {
+ if (entry.getValue().getLength() != pathInfoMap.get(entry.getKey()).getLength()) {
LOG.error(" Metadata file size mismatch " + entry.getKey() + ", size equality "
- + (entry.getValue().getLen() == fileStatusMap.get(entry.getKey()).getLen())
- + ". Metadata size " + entry.getValue().getLen() + ", FS size "
- + metadataFileStatusMap.get(entry.getKey()).getLen());
+ + (entry.getValue().getLength() == pathInfoMap.get(entry.getKey()).getLength())
+ + ". Metadata size " + entry.getValue().getLength() + ", FS size "
+ + metadataPathInfoMap.get(entry.getKey()).getLength());
}
}
}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
index 3e0ff6ce3ce..682dbbcd73a 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
@@ -18,7 +18,6 @@
package org.apache.hudi.cli.commands;
-import org.apache.spark.sql.hudi.DeDupeType;
import org.apache.hudi.cli.HoodieCLI;
import org.apache.hudi.cli.HoodiePrintHelper;
import org.apache.hudi.cli.HoodieTableHeaderFields;
@@ -36,10 +35,11 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.PartitionPathEncodeUtils;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.avro.AvroRuntimeException;
-import org.apache.hadoop.fs.Path;
import org.apache.spark.launcher.SparkLauncher;
+import org.apache.spark.sql.hudi.DeDupeType;
import org.apache.spark.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -122,22 +122,22 @@ public class RepairsCommand {
String latestCommit =
client.getActiveTimeline().getCommitTimeline().lastInstant().get().getTimestamp();
List<String> partitionPaths =
- FSUtils.getAllPartitionFoldersThreeLevelsDown(HoodieCLI.fs, client.getBasePath());
- Path basePath = new Path(client.getBasePath());
+ FSUtils.getAllPartitionFoldersThreeLevelsDown(HoodieCLI.storage, client.getBasePath());
+ StoragePath basePath = new StoragePath(client.getBasePath());
String[][] rows = new String[partitionPaths.size()][];
int ind = 0;
for (String partition : partitionPaths) {
- Path partitionPath = FSUtils.getPartitionPath(basePath, partition);
+ StoragePath partitionPath = FSUtils.getPartitionPath(basePath, partition);
String[] row = new String[3];
row[0] = partition;
row[1] = "Yes";
row[2] = "None";
- if (!HoodiePartitionMetadata.hasPartitionMetadata(HoodieCLI.fs, partitionPath)) {
+ if (!HoodiePartitionMetadata.hasPartitionMetadata(HoodieCLI.storage, partitionPath)) {
row[1] = "No";
if (!dryRun) {
HoodiePartitionMetadata partitionMetadata =
- new HoodiePartitionMetadata(HoodieCLI.fs, latestCommit, basePath, partitionPath,
+ new HoodiePartitionMetadata(HoodieCLI.storage, latestCommit, basePath, partitionPath,
client.getTableConfig().getPartitionMetafileFormat());
partitionMetadata.trySave(0);
row[2] = "Repaired";
@@ -163,13 +163,15 @@ public class RepairsCommand {
newProps.load(fileInputStream);
}
Map<String, String> oldProps = client.getTableConfig().propsMap();
- Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME);
- HoodieTableConfig.create(client.getFs(), metaPathDir, newProps);
+ StoragePath metaPathDir = new StoragePath(client.getBasePath(), METAFOLDER_NAME);
+ HoodieTableConfig.create(client.getStorage(), metaPathDir, newProps);
// reload new props as checksum would have been added
- newProps = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()).getTableConfig().getProps();
+ newProps =
+ HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()).getTableConfig().getProps();
TreeSet<String> allPropKeys = new TreeSet<>();
- allPropKeys.addAll(newProps.keySet().stream().map(Object::toString).collect(Collectors.toSet()));
+ allPropKeys.addAll(
+ newProps.keySet().stream().map(Object::toString).collect(Collectors.toSet()));
allPropKeys.addAll(oldProps.keySet());
String[][] rows = new String[allPropKeys.size()][];
@@ -197,11 +199,13 @@ public class RepairsCommand {
CleanerUtils.getCleanerPlan(client, instant);
} catch (AvroRuntimeException e) {
LOG.warn("Corruption found. Trying to remove corrupted clean instant file: " + instant);
- HoodieActiveTimeline.deleteInstantFile(client.getFs(), client.getMetaPath(), instant);
+ HoodieActiveTimeline.deleteInstantFile(client.getStorage(), client.getMetaPath(),
+ instant);
} catch (IOException ioe) {
if (ioe.getMessage().contains("Not an Avro data file")) {
LOG.warn("Corruption found. Trying to remove corrupted clean instant file: " + instant);
- HoodieActiveTimeline.deleteInstantFile(client.getFs(), client.getMetaPath(), instant);
+ HoodieActiveTimeline.deleteInstantFile(client.getStorage(), client.getMetaPath(),
+ instant);
} else {
throw new HoodieIOException(ioe.getMessage(), ioe);
}
@@ -225,16 +229,21 @@ public class RepairsCommand {
HoodieLocalEngineContext engineContext = new HoodieLocalEngineContext(HoodieCLI.conf);
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
- List<String> partitionPaths = FSUtils.getAllPartitionPaths(engineContext, client.getBasePath(), false);
- Path basePath = new Path(client.getBasePath());
+ List<String> partitionPaths =
+ FSUtils.getAllPartitionPaths(engineContext, client.getBasePath(), false);
+ StoragePath basePath = new StoragePath(client.getBasePath());
String[][] rows = new String[partitionPaths.size()][];
int ind = 0;
for (String partitionPath : partitionPaths) {
- Path partition = FSUtils.getPartitionPath(client.getBasePath(), partitionPath);
- Option<Path> textFormatFile = HoodiePartitionMetadata.textFormatMetaPathIfExists(HoodieCLI.fs, partition);
- Option<Path> baseFormatFile = HoodiePartitionMetadata.baseFormatMetaPathIfExists(HoodieCLI.fs, partition);
- String latestCommit = client.getActiveTimeline().getCommitTimeline().lastInstant().get().getTimestamp();
+ StoragePath partition =
+ FSUtils.getPartitionPath(client.getBasePath(), partitionPath);
+ Option<StoragePath> textFormatFile =
+ HoodiePartitionMetadata.textFormatMetaPathIfExists(HoodieCLI.storage, partition);
+ Option<StoragePath> baseFormatFile =
+ HoodiePartitionMetadata.baseFormatMetaPathIfExists(HoodieCLI.storage, partition);
+ String latestCommit =
+ client.getActiveTimeline().getCommitTimeline().lastInstant().get().getTimestamp();
String[] row = new String[] {
partitionPath,
@@ -245,15 +254,16 @@ public class RepairsCommand {
if (!dryRun) {
if (!baseFormatFile.isPresent()) {
- HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(HoodieCLI.fs, latestCommit, basePath, partition,
- Option.of(client.getTableConfig().getBaseFileFormat()));
+ HoodiePartitionMetadata partitionMetadata =
+ new HoodiePartitionMetadata(HoodieCLI.storage, latestCommit, basePath, partition,
+ Option.of(client.getTableConfig().getBaseFileFormat()));
partitionMetadata.trySave(0);
}
// delete it, in case we failed midway last time.
textFormatFile.ifPresent(path -> {
try {
- HoodieCLI.fs.delete(path, false);
+ HoodieCLI.storage.deleteFile(path);
} catch (IOException e) {
throw new HoodieIOException(e.getMessage(), e);
}
@@ -267,7 +277,7 @@ public class RepairsCommand {
Properties props = new Properties();
props.setProperty(HoodieTableConfig.PARTITION_METAFILE_USE_BASE_FORMAT.key(), "true");
- HoodieTableConfig.update(HoodieCLI.fs, new Path(client.getMetaPath()), props);
+ HoodieTableConfig.update(HoodieCLI.storage, new StoragePath(client.getMetaPath()), props);
return HoodiePrintHelper.print(new String[] {
HoodieTableHeaderFields.HEADER_PARTITION_PATH,
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
index ba3b404474b..31584ca0a7d 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
@@ -43,6 +43,8 @@ import org.apache.hudi.exception.HoodieSavepointException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.keygen.constant.KeyGeneratorType;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.action.compact.strategy.UnBoundedCompactionStrategy;
import org.apache.hudi.table.marker.WriteMarkersFactory;
@@ -381,8 +383,10 @@ public class SparkMain {
private static int deduplicatePartitionPath(JavaSparkContext jsc, String duplicatedPartitionPath,
String repairedOutputPath, String basePath, boolean dryRun, String dedupeType) {
- DedupeSparkJob job = new DedupeSparkJob(basePath, duplicatedPartitionPath, repairedOutputPath, new SQLContext(jsc),
- HadoopFSUtils.getFs(basePath, jsc.hadoopConfiguration()), DeDupeType.withName(dedupeType));
+ DedupeSparkJob job = new DedupeSparkJob(basePath, duplicatedPartitionPath, repairedOutputPath,
+ new SQLContext(jsc),
+ HoodieStorageUtils.getStorage(basePath, jsc.hadoopConfiguration()),
+ DeDupeType.withName(dedupeType));
job.fixDuplicates(dryRun);
return 0;
}
@@ -475,7 +479,7 @@ public class SparkMain {
String payloadClassName, String enableHiveSync, String propsFilePath, List<String> configs) throws IOException {
TypedProperties properties = propsFilePath == null ? buildProperties(configs)
- : readConfig(jsc.hadoopConfiguration(), new Path(propsFilePath), configs).getProps(true);
+ : readConfig(jsc.hadoopConfiguration(), new StoragePath(propsFilePath), configs).getProps(true);
properties.setProperty(HoodieBootstrapConfig.BASE_PATH.key(), sourcePath);
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java
index 2c8ab342f31..f8e60ba8cee 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java
@@ -18,12 +18,6 @@
package org.apache.hudi.cli.commands;
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.Snapshot;
-import com.codahale.metrics.UniformReservoir;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.cli.HoodieCLI;
import org.apache.hudi.cli.HoodiePrintHelper;
import org.apache.hudi.cli.HoodieTableHeaderFields;
@@ -34,6 +28,13 @@ import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.NumericUtils;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorage;
+
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Snapshot;
+import com.codahale.metrics.UniformReservoir;
import org.springframework.shell.standard.ShellComponent;
import org.springframework.shell.standard.ShellMethod;
import org.springframework.shell.standard.ShellOption;
@@ -113,16 +114,18 @@ public class StatsCommand {
defaultValue = "false") final boolean headerOnly)
throws IOException {
- FileSystem fs = HoodieCLI.fs;
- String globPath = String.format("%s/%s/*", HoodieCLI.getTableMetaClient().getBasePath(), globRegex);
- List<FileStatus> statuses = FSUtils.getGlobStatusExcludingMetaFolder(fs, new Path(globPath));
+ HoodieStorage storage = HoodieCLI.storage;
+ String globPath =
+ String.format("%s/%s/*", HoodieCLI.getTableMetaClient().getBasePath(), globRegex);
+ List<StoragePathInfo> pathInfoList = FSUtils.getGlobStatusExcludingMetaFolder(storage,
+ new StoragePath(globPath));
// max, min, #small files < 10MB, 50th, avg, 95th
Histogram globalHistogram = new Histogram(new UniformReservoir(MAX_FILES));
HashMap<String, Histogram> commitHistoMap = new HashMap<>();
- for (FileStatus fileStatus : statuses) {
- String instantTime = FSUtils.getCommitTime(fileStatus.getPath().getName());
- long sz = fileStatus.getLen();
+ for (StoragePathInfo pathInfo : pathInfoList) {
+ String instantTime = FSUtils.getCommitTime(pathInfo.getPath().getName());
+ long sz = pathInfo.getLength();
if (!commitHistoMap.containsKey(instantTime)) {
commitHistoMap.put(instantTime, new Histogram(new UniformReservoir(MAX_FILES)));
}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java
index b6256f74006..59dbdde04bc 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java
@@ -36,10 +36,10 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.TableNotFoundException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.action.compact.strategy.UnBoundedCompactionStrategy;
import org.apache.avro.Schema;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.shell.standard.ShellComponent;
@@ -172,7 +172,7 @@ public class TableCommand {
List<Comparable[]> rows = new ArrayList<>();
rows.add(new Comparable[] {"basePath", client.getBasePath()});
rows.add(new Comparable[] {"metaPath", client.getMetaPath()});
- rows.add(new Comparable[] {"fileSystem", client.getFs().getScheme()});
+ rows.add(new Comparable[] {"fileSystem", client.getStorage().getScheme()});
client.getTableConfig().propsMap().entrySet().forEach(e -> {
rows.add(new Comparable[] {e.getKey(), e.getValue()});
});
@@ -212,8 +212,8 @@ public class TableCommand {
public String recoverTableConfig() throws IOException {
HoodieCLI.refreshTableMetadata();
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
- Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME);
- HoodieTableConfig.recover(client.getFs(), metaPathDir);
+ StoragePath metaPathDir = new StoragePath(client.getBasePath(), METAFOLDER_NAME);
+ HoodieTableConfig.recover(client.getStorage(), metaPathDir);
return descTable();
}
@@ -228,8 +228,8 @@ public class TableCommand {
try (FileInputStream fileInputStream = new FileInputStream(updatePropsFilePath)) {
updatedProps.load(fileInputStream);
}
- Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME);
- HoodieTableConfig.update(client.getFs(), metaPathDir, updatedProps);
+ StoragePath metaPathDir = new StoragePath(client.getBasePath(), METAFOLDER_NAME);
+ HoodieTableConfig.update(client.getStorage(), metaPathDir, updatedProps);
HoodieCLI.refreshTableMetadata();
Map<String, String> newProps = HoodieCLI.getTableMetaClient().getTableConfig().propsMap();
@@ -244,8 +244,8 @@ public class TableCommand {
Map<String, String> oldProps = client.getTableConfig().propsMap();
Set<String> deleteConfigs = Arrays.stream(csConfigs.split(",")).collect(Collectors.toSet());
- Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME);
- HoodieTableConfig.delete(client.getFs(), metaPathDir, deleteConfigs);
+ StoragePath metaPathDir = new StoragePath(client.getBasePath(), METAFOLDER_NAME);
+ HoodieTableConfig.delete(client.getStorage(), metaPathDir, deleteConfigs);
HoodieCLI.refreshTableMetadata();
Map<String, String> newProps = HoodieCLI.getTableMetaClient().getTableConfig().propsMap();
@@ -358,15 +358,16 @@ public class TableCommand {
}
break;
default:
- throw new HoodieException("Unsupported change type " + changeType + ". Only support MOR or COW.");
+ throw new HoodieException(
+ "Unsupported change type " + changeType + ". Only support MOR or COW.");
}
Properties updatedProps = new Properties();
updatedProps.putAll(oldProps);
// change the table type to target type
updatedProps.put(HoodieTableConfig.TYPE.key(), targetType);
- Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME);
- HoodieTableConfig.update(client.getFs(), metaPathDir, updatedProps);
+ StoragePath metaPathDir = new StoragePath(client.getBasePath(), METAFOLDER_NAME);
+ HoodieTableConfig.update(client.getStorage(), metaPathDir, updatedProps);
HoodieCLI.refreshTableMetadata();
Map<String, String> newProps = HoodieCLI.getTableMetaClient().getTableConfig().propsMap();
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TimelineCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TimelineCommand.java
index 2b89175293d..063bc61e8c0 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TimelineCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TimelineCommand.java
@@ -32,10 +32,10 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.shell.standard.ShellComponent;
@@ -45,7 +45,6 @@ import org.springframework.shell.standard.ShellOption;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
@@ -85,13 +84,13 @@ public class TimelineCommand {
HoodieTableMetaClient mtMetaClient = getMetadataTableMetaClient(metaClient);
return printTimelineInfoWithMetadataTable(
metaClient.getActiveTimeline(), mtMetaClient.getActiveTimeline(),
- getInstantInfoFromTimeline(metaClient.getFs(), metaClient.getMetaPath()),
- getInstantInfoFromTimeline(mtMetaClient.getFs(), mtMetaClient.getMetaPath()),
+ getInstantInfoFromTimeline(metaClient.getStorage(), metaClient.getMetaPath()),
+ getInstantInfoFromTimeline(mtMetaClient.getStorage(), mtMetaClient.getMetaPath()),
limit, sortByField, descending, headerOnly, true, showTimeSeconds, showRollbackInfo);
}
return printTimelineInfo(
metaClient.getActiveTimeline(),
- getInstantInfoFromTimeline(metaClient.getFs(), metaClient.getMetaPath()),
+ getInstantInfoFromTimeline(metaClient.getStorage(), metaClient.getMetaPath()),
limit, sortByField, descending, headerOnly, true, showTimeSeconds, showRollbackInfo);
} catch (IOException e) {
e.printStackTrace();
@@ -114,7 +113,7 @@ public class TimelineCommand {
try {
return printTimelineInfo(
metaClient.getActiveTimeline().filterInflightsAndRequested(),
- getInstantInfoFromTimeline(metaClient.getFs(), metaClient.getMetaPath()),
+ getInstantInfoFromTimeline(metaClient.getStorage(), metaClient.getMetaPath()),
limit, sortByField, descending, headerOnly, true, showTimeSeconds, showRollbackInfo);
} catch (IOException e) {
e.printStackTrace();
@@ -136,7 +135,7 @@ public class TimelineCommand {
try {
return printTimelineInfo(
metaClient.getActiveTimeline(),
- getInstantInfoFromTimeline(metaClient.getFs(), metaClient.getMetaPath()),
+ getInstantInfoFromTimeline(metaClient.getStorage(), metaClient.getMetaPath()),
limit, sortByField, descending, headerOnly, true, showTimeSeconds, false);
} catch (IOException e) {
e.printStackTrace();
@@ -158,7 +157,7 @@ public class TimelineCommand {
try {
return printTimelineInfo(
metaClient.getActiveTimeline().filterInflightsAndRequested(),
- getInstantInfoFromTimeline(metaClient.getFs(), metaClient.getMetaPath()),
+ getInstantInfoFromTimeline(metaClient.getStorage(), metaClient.getMetaPath()),
limit, sortByField, descending, headerOnly, true, showTimeSeconds, false);
} catch (IOException e) {
e.printStackTrace();
@@ -175,14 +174,14 @@ public class TimelineCommand {
}
private Map<String, Map<HoodieInstant.State, HoodieInstantWithModTime>> getInstantInfoFromTimeline(
- FileSystem fs, String metaPath) throws IOException {
+ HoodieStorage storage, String metaPath) throws IOException {
Map<String, Map<HoodieInstant.State, HoodieInstantWithModTime>> instantMap = new HashMap<>();
- Stream<HoodieInstantWithModTime> instantStream = Arrays.stream(
- HoodieTableMetaClient.scanFiles(fs, new Path(metaPath), path -> {
+ Stream<HoodieInstantWithModTime> instantStream =
+ HoodieTableMetaClient.scanFiles(storage, new StoragePath(metaPath), path -> {
// Include only the meta files with extensions that needs to be included
String extension = HoodieInstant.getTimelineFileExtension(path.getName());
return HoodieActiveTimeline.VALID_EXTENSIONS_IN_ACTIVE_TIMELINE.contains(extension);
- })).map(HoodieInstantWithModTime::new);
+ }).stream().map(HoodieInstantWithModTime::new);
instantStream.forEach(instant -> {
instantMap.computeIfAbsent(instant.getTimestamp(), t -> new HashMap<>())
.put(instant.getState(), instant);
@@ -369,9 +368,9 @@ public class TimelineCommand {
private final long modificationTimeMs;
- public HoodieInstantWithModTime(FileStatus fileStatus) {
- super(fileStatus);
- this.modificationTimeMs = fileStatus.getModificationTime();
+ public HoodieInstantWithModTime(StoragePathInfo pathInfo) {
+ super(pathInfo);
+ this.modificationTimeMs = pathInfo.getModificationTime();
}
public long getModificationTime() {
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java
index 2df1dd3c2a3..178e1ec2309 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java
@@ -100,8 +100,7 @@ public class TestArchivedCommitsCommand extends CLIFunctionalTestHarness {
// Simulate a compaction commit in metadata table timeline
// so the archival in data table can happen
- HoodieTestUtils.createCompactionCommitInMetadataTable(
- hadoopConf(), metaClient.getFs(), tablePath, "105");
+ HoodieTestUtils.createCompactionCommitInMetadataTable(hadoopConf(), tablePath, "105");
metaClient = HoodieTableMetaClient.reload(metaClient);
// reload the timeline and get all the commits before archive
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java
index 038e219b155..5ffa2feb9db 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java
@@ -38,10 +38,10 @@ import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -93,8 +93,9 @@ public class TestCleansCommand extends CLIFunctionalTestHarness {
metaClient = HoodieCLI.getTableMetaClient();
String fileId1 = UUID.randomUUID().toString();
String fileId2 = UUID.randomUUID().toString();
- FileSystem fs = HadoopFSUtils.getFs(basePath(), hadoopConf());
- HoodieTestDataGenerator.writePartitionMetadataDeprecated(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
+ HoodieStorage storage = HoodieStorageUtils.getStorage(basePath(), hadoopConf());
+ HoodieTestDataGenerator.writePartitionMetadataDeprecated(storage,
+ HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
// Create four commits
for (int i = 100; i < 104; i++) {
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
index 01eca8a5e9b..d84b547504c 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
@@ -33,20 +33,21 @@ import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
-import org.apache.hudi.common.testutils.InProcessTimeGenerator;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.testutils.InProcessTimeGenerator;
import org.apache.hudi.common.util.NumericUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieArchivalConfig;
import org.apache.hudi.config.HoodieCleanConfig;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.table.HoodieSparkTable;
-import org.apache.hadoop.fs.FileSystem;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -161,9 +162,9 @@ public class TestCommitsCommand extends CLIFunctionalTestHarness {
}
private String generateExpectData(int records, Map<String, Integer[]> data) throws IOException {
- FileSystem fs = FileSystem.get(hadoopConf());
+ HoodieStorage storage = HoodieStorageUtils.getStorage(hadoopConf());
List<String> partitionPaths =
- FSUtils.getAllPartitionFoldersThreeLevelsDown(fs, tablePath1);
+ FSUtils.getAllPartitionFoldersThreeLevelsDown(storage, tablePath1);
int partitions = partitionPaths.size();
// default pre-commit is not null, file add always be 0 and update always be partition nums
@@ -302,7 +303,7 @@ public class TestCommitsCommand extends CLIFunctionalTestHarness {
if (enableMetadataTable) {
// Simulate a compaction commit in metadata table timeline
// so the archival in data table can happen
- createCompactionCommitInMetadataTable(hadoopConf(), metaClient.getFs(), tablePath1, "106");
+ createCompactionCommitInMetadataTable(hadoopConf(), tablePath1, "106");
}
// archive
@@ -336,7 +337,7 @@ public class TestCommitsCommand extends CLIFunctionalTestHarness {
if (enableMetadataTable) {
// Simulate a compaction commit in metadata table timeline
// so the archival in data table can happen
- createCompactionCommitInMetadataTable(hadoopConf(), metaClient.getFs(), tablePath1, "194");
+ createCompactionCommitInMetadataTable(hadoopConf(), tablePath1, "194");
}
for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCompactionCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCompactionCommand.java
index 8182c8ed734..b1035ff30ce 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCompactionCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCompactionCommand.java
@@ -25,8 +25,6 @@ import org.apache.hudi.cli.TableHeader;
import org.apache.hudi.cli.functional.CLIFunctionalTestHarness;
import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator;
import org.apache.hudi.client.timeline.HoodieTimelineArchiver;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
-import org.apache.hudi.hadoop.fs.NoOpConsistencyGuard;
import org.apache.hudi.common.model.HoodieAvroPayload;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -43,7 +41,6 @@ import org.apache.hudi.config.HoodieArchivalConfig;
import org.apache.hudi.config.HoodieCleanConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem;
import org.apache.hudi.table.HoodieSparkTable;
import org.junit.jupiter.api.BeforeEach;
@@ -164,9 +161,7 @@ public class TestCompactionCommand extends CLIFunctionalTestHarness {
});
// Simulate a compaction commit in metadata table timeline
// so the archival in data table can happen
- HoodieTestUtils.createCompactionCommitInMetadataTable(hadoopConf(),
- new HoodieWrapperFileSystem(
- HadoopFSUtils.getFs(tablePath, hadoopConf()), new NoOpConsistencyGuard()), tablePath, "007");
+ HoodieTestUtils.createCompactionCommitInMetadataTable(hadoopConf(), tablePath, "007");
}
private void generateArchive() throws IOException {
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestDiffCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestDiffCommand.java
index 1ce777c71b3..c1c1157702b 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestDiffCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestDiffCommand.java
@@ -37,10 +37,10 @@ import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.util.NumericUtils;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -92,8 +92,9 @@ public class TestDiffCommand extends CLIFunctionalTestHarness {
HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
String fileId1 = UUID.randomUUID().toString();
String fileId2 = UUID.randomUUID().toString();
- FileSystem fs = HadoopFSUtils.getFs(basePath(), hadoopConf());
- HoodieTestDataGenerator.writePartitionMetadataDeprecated(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
+ HoodieStorage storage = HoodieStorageUtils.getStorage(basePath(), hadoopConf());
+ HoodieTestDataGenerator.writePartitionMetadataDeprecated(storage,
+ HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
// Create four commits
Set<String> commits = new HashSet<>();
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java
index 74929e52526..55a5eeb1f75 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java
@@ -43,15 +43,14 @@ import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.testutils.SchemaTestUtil;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
@@ -65,7 +64,6 @@ import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -92,7 +90,7 @@ public class TestHoodieLogFileCommand extends CLIFunctionalTestHarness {
private HoodieAvroDataBlock dataBlock;
private HoodieCommandBlock commandBlock;
private String tablePath;
- private FileSystem fs;
+ private HoodieStorage storage;
private static final String INSTANT_TIME = "100";
@@ -109,12 +107,12 @@ public class TestHoodieLogFileCommand extends CLIFunctionalTestHarness {
"", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
Files.createDirectories(Paths.get(partitionPath));
- fs = HadoopFSUtils.getFs(tablePath, hadoopConf());
+ storage = HoodieStorageUtils.getStorage(tablePath, hadoopConf());
try (HoodieLogFormat.Writer writer = HoodieLogFormat.newWriterBuilder()
- .onParentPath(new Path(partitionPath))
+ .onParentPath(new StoragePath(partitionPath))
.withFileExtension(HoodieLogFile.DELTA_EXTENSION)
- .withFileId("test-log-fileid1").withDeltaCommit("100").withFs(fs)
+ .withFileId("test-log-fileid1").withDeltaCommit("100").withStorage(storage)
.withSizeThreshold(1).build()) {
// write data to file
@@ -137,7 +135,7 @@ public class TestHoodieLogFileCommand extends CLIFunctionalTestHarness {
@AfterEach
public void cleanUp() throws IOException {
- fs.close();
+ storage.close();
}
/**
@@ -209,9 +207,11 @@ public class TestHoodieLogFileCommand extends CLIFunctionalTestHarness {
try {
// set little threshold to split file.
writer =
- HoodieLogFormat.newWriterBuilder().onParentPath(new Path(partitionPath))
+ HoodieLogFormat.newWriterBuilder().onParentPath(new StoragePath(partitionPath))
.withFileExtension(HoodieLogFile.DELTA_EXTENSION)
- .withFileId("test-log-fileid1").withDeltaCommit(INSTANT_TIME).withFs(fs).withSizeThreshold(500).build();
+ .withFileId("test-log-fileid1").withDeltaCommit(INSTANT_TIME).withStorage(
+ storage)
+ .withSizeThreshold(500).build();
SchemaTestUtil testUtil = new SchemaTestUtil();
List<HoodieRecord> records1 = testUtil.generateHoodieTestRecords(0, 100).stream().map(HoodieAvroIndexedRecord::new).collect(Collectors.toList());
@@ -227,14 +227,15 @@ public class TestHoodieLogFileCommand extends CLIFunctionalTestHarness {
}
Object result = shell.evaluate(() -> "show logfile records --logFilePathPattern "
- + partitionPath + "/* --mergeRecords true");
+ + partitionPath + "/* --mergeRecords true");
assertTrue(ShellEvaluationResultUtil.isSuccess(result));
// get expected result of 10 records.
- List<String> logFilePaths = Arrays.stream(fs.globStatus(new Path(partitionPath + "/*")))
+ List<String> logFilePaths = storage.globEntries(new StoragePath(partitionPath + "/*"))
+ .stream()
.map(status -> status.getPath().toString()).collect(Collectors.toList());
HoodieMergedLogRecordScanner scanner = HoodieMergedLogRecordScanner.newBuilder()
- .withFileSystem(fs)
+ .withStorage(storage)
.withBasePath(tablePath)
.withLogFilePaths(logFilePaths)
.withReaderSchema(schema)
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
index f27ca62b3fb..09f753796f9 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
@@ -40,6 +40,7 @@ import org.apache.hudi.common.util.PartitionPathEncodeUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.keygen.SimpleKeyGenerator;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.testutils.Assertions;
import org.apache.avro.generic.GenericRecord;
@@ -140,7 +141,7 @@ public class TestRepairsCommand extends CLIFunctionalTestHarness {
assertTrue(ShellEvaluationResultUtil.isSuccess(result));
// expected all 'No'.
- String[][] rows = FSUtils.getAllPartitionFoldersThreeLevelsDown(fs, tablePath)
+ String[][] rows = FSUtils.getAllPartitionFoldersThreeLevelsDown(HoodieStorageUtils.getStorage(fs), tablePath)
.stream()
.map(partition -> new String[] {partition, "No", "None"})
.toArray(String[][]::new);
@@ -170,7 +171,7 @@ public class TestRepairsCommand extends CLIFunctionalTestHarness {
Object result = shell.evaluate(() -> "repair addpartitionmeta --dryrun false");
assertTrue(ShellEvaluationResultUtil.isSuccess(result));
- List<String> paths = FSUtils.getAllPartitionFoldersThreeLevelsDown(fs, tablePath);
+ List<String> paths = FSUtils.getAllPartitionFoldersThreeLevelsDown(HoodieStorageUtils.getStorage(fs), tablePath);
// after dry run, the action will be 'Repaired'
String[][] rows = paths.stream()
.map(partition -> new String[] {partition, "No", "Repaired"})
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
index 237a9f1985b..5211da14b18 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
@@ -31,9 +31,9 @@ import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.testutils.FileCreateUtils;
import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.testutils.HoodieClientTestUtils;
-import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
@@ -115,14 +115,18 @@ public class TestUpgradeDowngradeCommand extends CLIFunctionalTestHarness {
public void testUpgradeDowngradeCommand(HoodieTableVersion fromVersion, HoodieTableVersion toVersion) throws Exception {
// Start with hoodie.table.version to 5
metaClient.getTableConfig().setTableVersion(HoodieTableVersion.FIVE);
- try (OutputStream os = metaClient.getFs().create(new Path(metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE), true)) {
+ try (OutputStream os = metaClient.getStorage().create(
+ new StoragePath(
+ metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE),
+ true)) {
metaClient.getTableConfig().getProps().store(os, "");
}
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
// verify marker files for inflight commit exists
for (String partitionPath : DEFAULT_PARTITION_PATHS) {
- assertEquals(1, FileCreateUtils.getTotalMarkerFileCount(tablePath, partitionPath, "101", IOType.MERGE));
+ assertEquals(1,
+ FileCreateUtils.getTotalMarkerFileCount(tablePath, partitionPath, "101", IOType.MERGE));
}
if (fromVersion != HoodieTableVersion.FIVE) {
@@ -161,12 +165,15 @@ public class TestUpgradeDowngradeCommand extends CLIFunctionalTestHarness {
}
private void assertTableVersionFromPropertyFile(HoodieTableVersion expectedVersion) throws IOException {
- Path propertyFile = new Path(metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE);
+ StoragePath propertyFile =
+ new StoragePath(
+ metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE);
// Load the properties and verify
- InputStream inputStream = metaClient.getFs().open(propertyFile);
+ InputStream inputStream = metaClient.getStorage().open(propertyFile);
HoodieConfig config = new HoodieConfig();
config.getProps().load(inputStream);
inputStream.close();
- assertEquals(Integer.toString(expectedVersion.versionCode()), config.getString(HoodieTableConfig.VERSION));
+ assertEquals(Integer.toString(expectedVersion.versionCode()),
+ config.getString(HoodieTableConfig.VERSION));
}
}
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java
index 7786a57896e..4cfc6ca13d1 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java
@@ -80,10 +80,10 @@ public class ITTestHDFSParquetImportCommand extends HoodieCLIIntegrationTestBase
tablePath = basePath + StoragePath.SEPARATOR + tableName;
sourcePath = new Path(basePath, "source");
targetPath = new Path(tablePath);
- schemaFile = new Path(basePath, "file.schema").toString();
+ schemaFile = new StoragePath(basePath, "file.schema").toString();
// create schema file
- try (OutputStream schemaFileOS = fs.create(new Path(schemaFile))) {
+ try (OutputStream schemaFileOS = storage.create(new StoragePath(schemaFile))) {
schemaFileOS.write(getUTF8Bytes(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA));
}
@@ -171,17 +171,21 @@ public class ITTestHDFSParquetImportCommand extends HoodieCLIIntegrationTestBase
* Method to verify result is equals to expect.
*/
private void verifyResultData(List<GenericRecord> expectData) {
- Dataset<Row> ds = HoodieClientTestUtils.read(jsc, tablePath, sqlContext, fs, tablePath + "/*/*/*/*");
+ Dataset<Row> ds = HoodieClientTestUtils.read(jsc, tablePath, sqlContext,
+ storage, tablePath + "/*/*/*/*");
- List<Row> readData = ds.select("timestamp", "_row_key", "rider", "driver", "begin_lat", "begin_lon", "end_lat", "end_lon").collectAsList();
+ List<Row> readData =
+ ds.select("timestamp", "_row_key", "rider", "driver", "begin_lat", "begin_lon", "end_lat",
+ "end_lon").collectAsList();
List<HoodieTripModel> result = readData.stream().map(row ->
- new HoodieTripModel(row.getLong(0), row.getString(1), row.getString(2), row.getString(3), row.getDouble(4),
- row.getDouble(5), row.getDouble(6), row.getDouble(7)))
+ new HoodieTripModel(row.getLong(0), row.getString(1), row.getString(2), row.getString(3),
+ row.getDouble(4),
+ row.getDouble(5), row.getDouble(6), row.getDouble(7)))
.collect(Collectors.toList());
List<HoodieTripModel> expected = expectData.stream().map(g ->
- new HoodieTripModel(Long.parseLong(g.get("timestamp").toString()),
- g.get("_row_key").toString(),
+ new HoodieTripModel(Long.parseLong(g.get("timestamp").toString()),
+ g.get("_row_key").toString(),
g.get("rider").toString(),
g.get("driver").toString(),
Double.parseDouble(g.get("begin_lat").toString()),
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java
index d7e40c8f2f6..254be746e15 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java
@@ -18,9 +18,6 @@
package org.apache.hudi.cli.integ;
-import org.apache.avro.Schema;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.cli.HoodieCLI;
import org.apache.hudi.cli.commands.RepairsCommand;
@@ -36,14 +33,18 @@ import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.SchemaTestUtil;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.testutils.HoodieSparkWriteableTestTable;
+
+import org.apache.avro.Schema;
import org.apache.spark.sql.Dataset;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.shell.Shell;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.EnumSource;
import java.io.IOException;
import java.nio.file.Paths;
@@ -169,8 +170,9 @@ public class ITTestRepairsCommand extends HoodieCLIIntegrationTestBase {
// get fs and check number of latest files
HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient,
metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(),
- fs.listStatus(new Path(Paths.get(tablePath, duplicatedPartitionPath).toString())));
- List<String> filteredStatuses = fsView.getLatestBaseFiles().map(HoodieBaseFile::getPath).collect(Collectors.toList());
+ storage.listDirectEntries(new StoragePath(tablePath, duplicatedPartitionPath)));
+ List<String> filteredStatuses =
+ fsView.getLatestBaseFiles().map(HoodieBaseFile::getPath).collect(Collectors.toList());
assertEquals(3, filteredStatuses.size(), "There should be 3 files.");
// Before deduplicate, all files contain 210 records
@@ -186,8 +188,8 @@ public class ITTestRepairsCommand extends HoodieCLIIntegrationTestBase {
assertEquals(RepairsCommand.DEDUPLICATE_RETURN_PREFIX + repairedOutputPath, resultForCmd.toString());
// After deduplicate, there are 200 records
- FileStatus[] fileStatus = fs.listStatus(new Path(repairedOutputPath));
- files = Arrays.stream(fileStatus).map(status -> status.getPath().toString()).toArray(String[]::new);
+ List<StoragePathInfo> pathInfoList = storage.listDirectEntries(new StoragePath(repairedOutputPath));
+ files = pathInfoList.stream().map(status -> status.getPath().toString()).toArray(String[]::new);
Dataset result = readFiles(files);
assertEquals(200, result.count());
}
@@ -199,8 +201,10 @@ public class ITTestRepairsCommand extends HoodieCLIIntegrationTestBase {
connectTableAndReloadMetaClient(tablePath);
HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient,
metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(),
- fs.listStatus(new Path(Paths.get(tablePath, duplicatedPartitionPathWithUpdates).toString())));
- List<String> filteredStatuses = fsView.getLatestBaseFiles().map(HoodieBaseFile::getPath).collect(Collectors.toList());
+ storage.listDirectEntries(
+ new StoragePath(Paths.get(tablePath, duplicatedPartitionPathWithUpdates).toString())));
+ List<String> filteredStatuses =
+ fsView.getLatestBaseFiles().map(HoodieBaseFile::getPath).collect(Collectors.toList());
assertEquals(2, filteredStatuses.size(), "There should be 2 files.");
// Before deduplicate, all files contain 110 records
@@ -216,8 +220,8 @@ public class ITTestRepairsCommand extends HoodieCLIIntegrationTestBase {
assertEquals(RepairsCommand.DEDUPLICATE_RETURN_PREFIX + repairedOutputPath, resultForCmd.toString());
// After deduplicate, there are 100 records
- FileStatus[] fileStatus = fs.listStatus(new Path(repairedOutputPath));
- files = Arrays.stream(fileStatus).map(status -> status.getPath().toString()).toArray(String[]::new);
+ List<StoragePathInfo> pathInfoList = storage.listDirectEntries(new StoragePath(repairedOutputPath));
+ files = pathInfoList.stream().map(status -> status.getPath().toString()).toArray(String[]::new);
Dataset result = readFiles(files);
assertEquals(100, result.count());
}
@@ -229,8 +233,10 @@ public class ITTestRepairsCommand extends HoodieCLIIntegrationTestBase {
connectTableAndReloadMetaClient(tablePath);
HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient,
metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(),
- fs.listStatus(new Path(Paths.get(tablePath, duplicatedPartitionPathWithUpserts).toString())));
- List<String> filteredStatuses = fsView.getLatestBaseFiles().map(HoodieBaseFile::getPath).collect(Collectors.toList());
+ storage.listDirectEntries(
+ new StoragePath(Paths.get(tablePath, duplicatedPartitionPathWithUpserts).toString())));
+ List<String> filteredStatuses =
+ fsView.getLatestBaseFiles().map(HoodieBaseFile::getPath).collect(Collectors.toList());
assertEquals(3, filteredStatuses.size(), "There should be 3 files.");
// Before deduplicate, all files contain 120 records
@@ -246,8 +252,8 @@ public class ITTestRepairsCommand extends HoodieCLIIntegrationTestBase {
assertEquals(RepairsCommand.DEDUPLICATE_RETURN_PREFIX + repairedOutputPath, resultForCmd.toString());
// After deduplicate, there are 100 records
- FileStatus[] fileStatus = fs.listStatus(new Path(repairedOutputPath));
- files = Arrays.stream(fileStatus).map(status -> status.getPath().toString()).toArray(String[]::new);
+ List<StoragePathInfo> pathInfoList = storage.listDirectEntries(new StoragePath(repairedOutputPath));
+ files = pathInfoList.stream().map(status -> status.getPath().toString()).toArray(String[]::new);
Dataset result = readFiles(files);
assertEquals(100, result.count());
}
@@ -262,8 +268,9 @@ public class ITTestRepairsCommand extends HoodieCLIIntegrationTestBase {
connectTableAndReloadMetaClient(tablePath);
HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient,
metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(),
- fs.listStatus(new Path(Paths.get(tablePath, duplicatedNoPartitionPath).toString())));
- List<String> filteredStatuses = fsView.getLatestBaseFiles().map(HoodieBaseFile::getPath).collect(Collectors.toList());
+ storage.listDirectEntries(new StoragePath(tablePath, duplicatedNoPartitionPath)));
+ List<String> filteredStatuses =
+ fsView.getLatestBaseFiles().map(HoodieBaseFile::getPath).collect(Collectors.toList());
assertEquals(2, filteredStatuses.size(), "There should be 2 files.");
// Before deduplicate, all files contain 110 records
@@ -279,8 +286,8 @@ public class ITTestRepairsCommand extends HoodieCLIIntegrationTestBase {
assertEquals(RepairsCommand.DEDUPLICATE_RETURN_PREFIX + repairedOutputPath, resultForCmd.toString());
// After deduplicate, there are 100 records
- FileStatus[] fileStatus = fs.listStatus(new Path(repairedOutputPath));
- files = Arrays.stream(fileStatus).map(status -> status.getPath().toString()).toArray(String[]::new);
+ List<StoragePathInfo> pathInfoList = storage.listDirectEntries(new StoragePath(repairedOutputPath));
+ files = pathInfoList.stream().map(status -> status.getPath().toString()).toArray(String[]::new);
Dataset result = readFiles(files);
assertEquals(100, result.count());
}
@@ -296,8 +303,10 @@ public class ITTestRepairsCommand extends HoodieCLIIntegrationTestBase {
// get fs and check number of latest files
HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient,
metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(),
- fs.listStatus(new Path(Paths.get(tablePath, duplicatedPartitionPath).toString())));
- List<String> filteredStatuses = fsView.getLatestBaseFiles().map(HoodieBaseFile::getPath).collect(Collectors.toList());
+ storage.listDirectEntries(
+ new StoragePath(Paths.get(tablePath, duplicatedPartitionPath).toString())));
+ List<String> filteredStatuses =
+ fsView.getLatestBaseFiles().map(HoodieBaseFile::getPath).collect(Collectors.toList());
assertEquals(3, filteredStatuses.size(), "There should be 3 files.");
// Before deduplicate, all files contain 210 records
@@ -313,8 +322,9 @@ public class ITTestRepairsCommand extends HoodieCLIIntegrationTestBase {
assertEquals(RepairsCommand.DEDUPLICATE_RETURN_PREFIX + partitionPath, resultForCmd.toString());
// After deduplicate, there are 200 records under partition path
- FileStatus[] fileStatus = fs.listStatus(new Path(Paths.get(tablePath, duplicatedPartitionPath).toString()));
- files = Arrays.stream(fileStatus).map(status -> status.getPath().toString()).toArray(String[]::new);
+ List<StoragePathInfo> pathInfoList =
+ storage.listDirectEntries(new StoragePath(tablePath, duplicatedPartitionPath));
+ files = pathInfoList.stream().map(status -> status.getPath().toString()).toArray(String[]::new);
Dataset result = readFiles(files);
assertEquals(200, result.count());
}
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java
index 06a9662b1a1..673915efbfa 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java
@@ -35,7 +35,6 @@ import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
import org.apache.hudi.storage.StoragePath;
-import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@@ -154,13 +153,14 @@ public class ITTestSavepointsCommand extends HoodieCLIIntegrationTestBase {
HoodieTestDataGenerator.createSavepointFile(tablePath, savepoint, jsc.hadoopConfiguration());
// re-bootstrap metadata table
- Path metadataTableBasePath = new Path(HoodieTableMetadata.getMetadataTableBasePath(HoodieCLI.basePath));
+ StoragePath metadataTableBasePath =
+ new StoragePath(HoodieTableMetadata.getMetadataTableBasePath(HoodieCLI.basePath));
// then bootstrap metadata table at instant 104
HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(HoodieCLI.basePath)
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).build()).build();
SparkHoodieBackedTableMetadataWriter.create(HoodieCLI.conf, writeConfig, new HoodieSparkEngineContext(jsc)).close();
- assertTrue(HoodieCLI.fs.exists(metadataTableBasePath));
+ assertTrue(HoodieCLI.storage.exists(metadataTableBasePath));
// roll back to savepoint
Object result = shell.evaluate(() ->
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
index 009fbf140f0..d7c6ae9660e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
@@ -41,14 +41,14 @@ import org.apache.hudi.exception.HoodieCommitException;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieWriteConflictException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.metrics.HoodieMetrics;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.table.HoodieTable;
import com.codahale.metrics.Timer;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,7 +67,7 @@ public abstract class BaseHoodieClient implements Serializable, AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(BaseHoodieClient.class);
private static final long serialVersionUID = 1L;
- protected final transient FileSystem fs;
+ protected final transient HoodieStorage storage;
protected final transient HoodieEngineContext context;
protected final transient Configuration hadoopConf;
protected final transient HoodieMetrics metrics;
@@ -92,16 +92,17 @@ public abstract class BaseHoodieClient implements Serializable, AutoCloseable {
protected BaseHoodieClient(HoodieEngineContext context, HoodieWriteConfig clientConfig,
Option<EmbeddedTimelineService> timelineServer) {
this.hadoopConf = context.getHadoopConf().get();
- this.fs = HadoopFSUtils.getFs(clientConfig.getBasePath(), hadoopConf);
+ this.storage = HoodieStorageUtils.getStorage(clientConfig.getBasePath(), hadoopConf);
this.context = context;
this.basePath = clientConfig.getBasePath();
this.config = clientConfig;
this.timelineServer = timelineServer;
shouldStopTimelineServer = !timelineServer.isPresent();
- this.heartbeatClient = new HoodieHeartbeatClient(this.fs, this.basePath,
- clientConfig.getHoodieClientHeartbeatIntervalInMs(), clientConfig.getHoodieClientHeartbeatTolerableMisses());
+ this.heartbeatClient = new HoodieHeartbeatClient(storage, this.basePath,
+ clientConfig.getHoodieClientHeartbeatIntervalInMs(),
+ clientConfig.getHoodieClientHeartbeatTolerableMisses());
this.metrics = new HoodieMetrics(config);
- this.txnManager = new TransactionManager(config, fs);
+ this.txnManager = new TransactionManager(config, storage);
this.timeGenerator = TimeGenerators.getTimeGenerator(config.getTimeGeneratorConfig(), hadoopConf);
startEmbeddedServerView();
initWrapperFSMetrics();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
index 228eaf4d554..d73886319c6 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
@@ -962,11 +962,11 @@ public abstract class BaseHoodieTableServiceClient<I, T, O> extends BaseHoodieCl
HoodieTimeline.FULL_BOOTSTRAP_INSTANT_TS)) {
// do we need to handle failed rollback of a bootstrap
rollbackFailedBootstrap();
- HeartbeatUtils.deleteHeartbeatFile(fs, basePath, entry.getKey(), config);
+ HeartbeatUtils.deleteHeartbeatFile(storage, basePath, entry.getKey(), config);
break;
} else {
rollback(entry.getKey(), entry.getValue(), skipLocking);
- HeartbeatUtils.deleteHeartbeatFile(fs, basePath, entry.getKey(), config);
+ HeartbeatUtils.deleteHeartbeatFile(storage, basePath, entry.getKey(), config);
}
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
index 42545c826a4..85b09a0c516 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
@@ -39,10 +39,10 @@ import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.table.action.compact.OperationResult;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,15 +112,16 @@ public class CompactionAdminClient extends BaseHoodieClient {
if (!dryRun) {
// Overwrite compaction request with empty compaction operations
HoodieInstant inflight = new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, compactionInstant);
- Path inflightPath = new Path(metaClient.getMetaPath(), inflight.getFileName());
- if (metaClient.getFs().exists(inflightPath)) {
+ StoragePath inflightPath = new StoragePath(metaClient.getMetaPath(), inflight.getFileName());
+ if (metaClient.getStorage().exists(inflightPath)) {
// We need to rollback data-files because of this inflight compaction before unscheduling
throw new IllegalStateException("Please rollback the inflight compaction before unscheduling");
}
// Leave the trace in aux folder but delete from metapath.
// TODO: Add a rollback instant but for compaction
HoodieInstant instant = new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, compactionInstant);
- boolean deleted = metaClient.getFs().delete(new Path(metaClient.getMetaPath(), instant.getFileName()), false);
+ boolean deleted = metaClient.getStorage().deleteFile(
+ new StoragePath(metaClient.getMetaPath(), instant.getFileName()));
ValidationUtils.checkArgument(deleted, "Unable to delete compaction instant.");
}
return new ArrayList<>();
@@ -146,8 +147,8 @@ public class CompactionAdminClient extends BaseHoodieClient {
CompactionUtils.getAllPendingCompactionOperations(metaClient).get(fgId);
HoodieCompactionPlan plan =
CompactionUtils.getCompactionPlan(metaClient, compactionOperationWithInstant.getKey());
- List<HoodieCompactionOperation> newOps = plan.getOperations().stream().filter(
- op -> (!op.getFileId().equals(fgId.getFileId())) && (!op.getPartitionPath().equals(fgId.getPartitionPath())))
+ List<HoodieCompactionOperation> newOps = plan.getOperations().stream().filter(op ->
+ (!op.getFileId().equals(fgId.getFileId())) && (!op.getPartitionPath().equals(fgId.getPartitionPath())))
.collect(Collectors.toList());
if (newOps.size() == plan.getOperations().size()) {
return new ArrayList<>();
@@ -156,8 +157,8 @@ public class CompactionAdminClient extends BaseHoodieClient {
HoodieCompactionPlan.newBuilder().setOperations(newOps).setExtraMetadata(plan.getExtraMetadata()).build();
HoodieInstant inflight =
new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, compactionOperationWithInstant.getLeft());
- Path inflightPath = new Path(metaClient.getMetaPath(), inflight.getFileName());
- if (metaClient.getFs().exists(inflightPath)) {
+ StoragePath inflightPath = new StoragePath(metaClient.getMetaPath(), inflight.getFileName());
+ if (metaClient.getStorage().exists(inflightPath)) {
// revert if in inflight state
metaClient.getActiveTimeline().revertInstantFromInflightToRequested(inflight);
}
@@ -206,12 +207,14 @@ public class CompactionAdminClient extends BaseHoodieClient {
*/
protected static void renameLogFile(HoodieTableMetaClient metaClient, HoodieLogFile oldLogFile,
HoodieLogFile newLogFile) throws IOException {
- FileStatus[] statuses = metaClient.getFs().listStatus(oldLogFile.getPath());
- ValidationUtils.checkArgument(statuses.length == 1, "Only one status must be present");
- ValidationUtils.checkArgument(statuses[0].isFile(), "Source File must exist");
- ValidationUtils.checkArgument(oldLogFile.getPath().getParent().equals(newLogFile.getPath().getParent()),
+ List<StoragePathInfo> pathInfoList =
+ metaClient.getStorage().listDirectEntries(oldLogFile.getPath());
+ ValidationUtils.checkArgument(pathInfoList.size() == 1, "Only one status must be present");
+ ValidationUtils.checkArgument(pathInfoList.get(0).isFile(), "Source File must exist");
+ ValidationUtils.checkArgument(
+ oldLogFile.getPath().getParent().equals(newLogFile.getPath().getParent()),
"Log file must only be moved within the parent directory");
- metaClient.getFs().rename(oldLogFile.getPath(), newLogFile.getPath());
+ metaClient.getStorage().rename(oldLogFile.getPath(), newLogFile.getPath());
}
/**
@@ -236,10 +239,10 @@ public class CompactionAdminClient extends BaseHoodieClient {
FileSlice fs = fileSliceOptional.get();
Option<HoodieBaseFile> df = fs.getBaseFile();
if (operation.getDataFileName().isPresent()) {
- String expPath = metaClient.getFs()
- .getFileStatus(
- new Path(FSUtils.getPartitionPath(metaClient.getBasePath(), operation.getPartitionPath()),
- new Path(operation.getDataFileName().get())))
+ String expPath = metaClient.getStorage()
+ .getPathInfo(new StoragePath(
+ FSUtils.getPartitionPath(metaClient.getBasePath(), operation.getPartitionPath()),
+ operation.getDataFileName().get()))
.getPath().toString();
ValidationUtils.checkArgument(df.isPresent(),
"Data File must be present. File Slice was : " + fs + ", operation :" + operation);
@@ -249,10 +252,11 @@ public class CompactionAdminClient extends BaseHoodieClient {
Set<HoodieLogFile> logFilesInFileSlice = fs.getLogFiles().collect(Collectors.toSet());
Set<HoodieLogFile> logFilesInCompactionOp = operation.getDeltaFileNames().stream().map(dp -> {
try {
- FileStatus[] fileStatuses = metaClient.getFs().listStatus(new Path(
- FSUtils.getPartitionPath(metaClient.getBasePath(), operation.getPartitionPath()), new Path(dp)));
- ValidationUtils.checkArgument(fileStatuses.length == 1, "Expect only 1 file-status");
- return new HoodieLogFile(fileStatuses[0]);
+ List<StoragePathInfo> pathInfoList = metaClient.getStorage()
+ .listDirectEntries(new StoragePath(
+ FSUtils.getPartitionPath(metaClient.getBasePath(), operation.getPartitionPath()), dp));
+ ValidationUtils.checkArgument(pathInfoList.size() == 1, "Expect only 1 file-status");
+ return new HoodieLogFile(pathInfoList.get(0));
} catch (FileNotFoundException fe) {
throw new CompactionValidationException(fe.getMessage());
} catch (IOException ioe) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
index ee5b0b065d9..7d2d2c505d6 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
@@ -27,11 +27,11 @@ import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.table.view.FileSystemViewStorageType;
import org.apache.hudi.common.util.NetworkUtils;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.timeline.service.TimelineService;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -182,7 +182,7 @@ public class EmbeddedTimelineService {
this.serviceConfig = timelineServiceConfBuilder.build();
server = timelineServiceCreator.create(context, hadoopConf.newCopy(), serviceConfig,
- HadoopFSUtils.getFs(writeConfig.getBasePath(), hadoopConf.newCopy()), viewManager);
+ HoodieStorageUtils.getStorage(writeConfig.getBasePath(), hadoopConf.newCopy()), viewManager);
serverPort = server.startService();
LOG.info("Started embedded timeline server at " + hostAddr + ":" + serverPort);
}
@@ -190,7 +190,7 @@ public class EmbeddedTimelineService {
@FunctionalInterface
interface TimelineServiceCreator {
TimelineService create(HoodieEngineContext context, Configuration hadoopConf, TimelineService.Config timelineServerConf,
- FileSystem fileSystem, FileSystemViewManager globalFileSystemViewManager) throws IOException;
+ HoodieStorage storage, FileSystemViewManager globalFileSystemViewManager) throws IOException;
}
private void setHostAddr(String embeddedTimelineServiceHostAddr) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java
index de54d880632..e7e8e6c1b5a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java
@@ -23,10 +23,9 @@ import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,16 +42,19 @@ public class HeartbeatUtils {
/**
* Deletes the heartbeat file for the specified instant.
- * @param fs
- * @param basePath
- * @param instantTime
- * @return
+ *
+ * @param storage {@link HoodieStorage} instance.
+ * @param basePath Hudi table base path.
+ * @param instantTime commit instant time.
+ * @return whether the file is successfully deleted.
*/
- public static boolean deleteHeartbeatFile(FileSystem fs, String basePath, String instantTime) {
+ public static boolean deleteHeartbeatFile(HoodieStorage storage,
+ String basePath,
+ String instantTime) {
boolean deleted = false;
try {
String heartbeatFolderPath = HoodieTableMetaClient.getHeartbeatFolderPath(basePath);
- deleted = fs.delete(new Path(heartbeatFolderPath + StoragePath.SEPARATOR + instantTime), false);
+ deleted = storage.deleteFile(new StoragePath(heartbeatFolderPath + StoragePath.SEPARATOR + instantTime));
if (!deleted) {
LOG.error("Failed to delete heartbeat for instant " + instantTime);
} else {
@@ -66,15 +68,19 @@ public class HeartbeatUtils {
/**
* Deletes the heartbeat file for the specified instant.
- * @param fs Hadoop FileSystem instance
- * @param basePath Hoodie table base path
+ *
+ * @param storage {@link HoodieStorage} instance.
+ * @param basePath Hoodie table base path
* @param instantTime Commit instant time
- * @param config HoodieWriteConfig instance
+ * @param config HoodieWriteConfig instance
* @return Boolean indicating whether heartbeat file was deleted or not
*/
- public static boolean deleteHeartbeatFile(FileSystem fs, String basePath, String instantTime, HoodieWriteConfig config) {
+ public static boolean deleteHeartbeatFile(HoodieStorage storage,
+ String basePath,
+ String instantTime,
+ HoodieWriteConfig config) {
if (config.getFailedWritesCleanPolicy().isLazy()) {
- return deleteHeartbeatFile(fs, basePath, instantTime);
+ return deleteHeartbeatFile(storage, basePath, instantTime);
}
return false;
@@ -92,8 +98,10 @@ public class HeartbeatUtils {
ValidationUtils.checkArgument(heartbeatClient != null);
try {
if (config.getFailedWritesCleanPolicy().isLazy() && heartbeatClient.isHeartbeatExpired(instantTime)) {
- throw new HoodieException("Heartbeat for instant " + instantTime + " has expired, last heartbeat "
- + getLastHeartbeatTime(table.getMetaClient().getFs(), config.getBasePath(), instantTime));
+ throw new HoodieException(
+ "Heartbeat for instant " + instantTime + " has expired, last heartbeat "
+ + getLastHeartbeatTime(
+ table.getMetaClient().getStorage(), config.getBasePath(), instantTime));
}
} catch (IOException io) {
throw new HoodieException("Unable to read heartbeat", io);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java
index 0b1c607c51f..460ebdfd11e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java
@@ -23,9 +23,8 @@ import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieHeartbeatException;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,7 +50,7 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
private static final Logger LOG = LoggerFactory.getLogger(HoodieHeartbeatClient.class);
- private final transient FileSystem fs;
+ private final transient HoodieStorage storage;
private final String basePath;
// path to the heartbeat folder where all writers are updating their heartbeats
private final String heartbeatFolderPath;
@@ -60,10 +59,10 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
private final Long maxAllowableHeartbeatIntervalInMs;
private final Map<String, Heartbeat> instantToHeartbeatMap;
- public HoodieHeartbeatClient(FileSystem fs, String basePath, Long heartbeatIntervalInMs,
+ public HoodieHeartbeatClient(HoodieStorage storage, String basePath, Long heartbeatIntervalInMs,
Integer numTolerableHeartbeatMisses) {
ValidationUtils.checkArgument(heartbeatIntervalInMs >= 1000, "Cannot set heartbeat lower than 1 second");
- this.fs = fs;
+ this.storage = storage;
this.basePath = basePath;
this.heartbeatFolderPath = HoodieTableMetaClient.getHeartbeatFolderPath(basePath);
this.heartbeatIntervalInMs = heartbeatIntervalInMs;
@@ -189,7 +188,7 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
Heartbeat heartbeat = instantToHeartbeatMap.get(instantTime);
if (isHeartbeatStarted(heartbeat)) {
stopHeartbeatTimer(heartbeat);
- HeartbeatUtils.deleteHeartbeatFile(fs, basePath, instantTime);
+ HeartbeatUtils.deleteHeartbeatFile(storage, basePath, instantTime);
LOG.info("Deleted heartbeat file for instant " + instantTime);
}
}
@@ -226,10 +225,10 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
LOG.info("Stopped heartbeat for instant " + heartbeat.getInstantTime());
}
- public static Boolean heartbeatExists(FileSystem fs, String basePath, String instantTime) throws IOException {
- Path heartbeatFilePath = new Path(HoodieTableMetaClient.getHeartbeatFolderPath(basePath)
- + StoragePath.SEPARATOR + instantTime);
- return fs.exists(heartbeatFilePath);
+ public static Boolean heartbeatExists(HoodieStorage storage, String basePath, String instantTime) throws IOException {
+ StoragePath heartbeatFilePath = new StoragePath(
+ HoodieTableMetaClient.getHeartbeatFolderPath(basePath) + StoragePath.SEPARATOR + instantTime);
+ return storage.exists(heartbeatFilePath);
}
public boolean isHeartbeatExpired(String instantTime) throws IOException {
@@ -237,7 +236,7 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
Heartbeat lastHeartbeatForWriter = instantToHeartbeatMap.get(instantTime);
if (lastHeartbeatForWriter == null) {
LOG.info("Heartbeat not found in internal map, falling back to reading from DFS");
- long lastHeartbeatForWriterTime = getLastHeartbeatTime(this.fs, basePath, instantTime);
+ long lastHeartbeatForWriterTime = getLastHeartbeatTime(this.storage, basePath, instantTime);
lastHeartbeatForWriter = new Heartbeat();
lastHeartbeatForWriter.setLastHeartbeatTime(lastHeartbeatForWriterTime);
lastHeartbeatForWriter.setInstantTime(instantTime);
@@ -255,7 +254,8 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
try {
Long newHeartbeatTime = System.currentTimeMillis();
OutputStream outputStream =
- this.fs.create(new Path(heartbeatFolderPath + StoragePath.SEPARATOR + instantTime), true);
+ this.storage.create(
+ new StoragePath(heartbeatFolderPath + StoragePath.SEPARATOR + instantTime), true);
outputStream.close();
Heartbeat heartbeat = instantToHeartbeatMap.get(instantTime);
if (heartbeat.getLastHeartbeatTime() != null && isHeartbeatExpired(instantTime)) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java
index 26bd964e4e9..1aed5092754 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java
@@ -81,7 +81,7 @@ public class HoodieTimelineArchiver<T extends HoodieAvroPayload, I, K, O> {
this.config = config;
this.table = table;
this.metaClient = table.getMetaClient();
- this.txnManager = new TransactionManager(config, table.getMetaClient().getFs());
+ this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
this.timelineWriter = LSMTimelineWriter.getInstance(config, table);
Pair<Integer, Integer> minAndMaxInstants = getMinAndMaxInstantsToKeep(table, metaClient);
this.minInstantsToKeep = minAndMaxInstants.getLeft();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LSMTimelineWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LSMTimelineWriter.java
index e5acf775a19..a1582c6e4c7 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LSMTimelineWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/LSMTimelineWriter.java
@@ -21,7 +21,6 @@ package org.apache.hudi.client.timeline;
import org.apache.hudi.avro.model.HoodieLSMTimelineInstant;
import org.apache.hudi.common.engine.TaskContextSupplier;
-import org.apache.hudi.common.table.timeline.MetadataConversionUtils;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
@@ -31,6 +30,7 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.ActiveAction;
import org.apache.hudi.common.table.timeline.LSMTimeline;
+import org.apache.hudi.common.table.timeline.MetadataConversionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.VisibleForTesting;
@@ -42,17 +42,16 @@ import org.apache.hudi.io.storage.HoodieAvroParquetReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.io.storage.HoodieFileWriter;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -108,7 +107,7 @@ public class LSMTimelineWriter {
Option<Consumer<ActiveAction>> preWriteCallback,
Option<Consumer<Exception>> exceptionHandler) throws HoodieCommitException {
ValidationUtils.checkArgument(!activeActions.isEmpty(), "The instant actions to write should not be empty");
- Path filePath = new Path(metaClient.getArchivePath(),
+ StoragePath filePath = new StoragePath(metaClient.getArchivePath(),
newFileName(activeActions.get(0).getInstantTime(), activeActions.get(activeActions.size() - 1).getInstantTime(), FILE_LAYER_ZERO));
try (HoodieFileWriter writer = openWriter(filePath)) {
Schema wrapperSchema = HoodieLSMTimelineInstant.getClassSchema();
@@ -177,17 +176,17 @@ public class LSMTimelineWriter {
// version starts from 1 and increases monotonically
int newVersion = currentVersion < 0 ? 1 : currentVersion + 1;
// create manifest file
- final Path manifestFilePath = LSMTimeline.getManifestFilePath(metaClient, newVersion);
- metaClient.getFs().createImmutableFileInPath(manifestFilePath, Option.of(content));
+ final StoragePath manifestFilePath = LSMTimeline.getManifestFilePath(metaClient, newVersion);
+ metaClient.getStorage().createImmutableFileInPath(manifestFilePath, Option.of(content));
// update version file
updateVersionFile(newVersion);
}
private void updateVersionFile(int newVersion) throws IOException {
byte[] content = getUTF8Bytes(String.valueOf(newVersion));
- final Path versionFilePath = LSMTimeline.getVersionFilePath(metaClient);
- metaClient.getFs().delete(versionFilePath, false);
- metaClient.getFs().createImmutableFileInPath(versionFilePath, Option.of(content));
+ final StoragePath versionFilePath = LSMTimeline.getVersionFilePath(metaClient);
+ metaClient.getStorage().deleteFile(versionFilePath);
+ metaClient.getStorage().createImmutableFileInPath(versionFilePath, Option.of(content));
}
/**
@@ -267,11 +266,11 @@ public class LSMTimelineWriter {
public void compactFiles(List<String> candidateFiles, String compactedFileName) {
LOG.info("Starting to compact source files.");
- try (HoodieFileWriter writer = openWriter(new Path(metaClient.getArchivePath(), compactedFileName))) {
+ try (HoodieFileWriter writer = openWriter(new StoragePath(metaClient.getArchivePath(), compactedFileName))) {
for (String fileName : candidateFiles) {
// Read the input source file
try (HoodieAvroParquetReader reader = (HoodieAvroParquetReader) HoodieFileReaderFactory.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
- .getFileReader(config, metaClient.getHadoopConf(), new Path(metaClient.getArchivePath(), fileName))) {
+ .getFileReader(config, metaClient.getHadoopConf(), new StoragePath(metaClient.getArchivePath(), fileName))) {
// Read the meta entry
try (ClosableIterator<IndexedRecord> iterator = reader.getIndexedRecordIterator(HoodieLSMTimelineInstant.getClassSchema(), HoodieLSMTimelineInstant.getClassSchema())) {
while (iterator.hasNext()) {
@@ -303,23 +302,27 @@ public class LSMTimelineWriter {
.collect(Collectors.toSet());
// delete the manifest file first
List<String> manifestFilesToClean = new ArrayList<>();
- Arrays.stream(LSMTimeline.listAllManifestFiles(metaClient)).forEach(fileStatus -> {
- if (!versionsToKeep.contains(LSMTimeline.getManifestVersion(fileStatus.getPath().getName()))) {
+ LSMTimeline.listAllManifestFiles(metaClient).forEach(fileStatus -> {
+ if (!versionsToKeep.contains(
+ LSMTimeline.getManifestVersion(fileStatus.getPath().getName()))) {
manifestFilesToClean.add(fileStatus.getPath().toString());
}
});
- FSUtils.deleteFilesParallelize(metaClient, manifestFilesToClean, context, config.getArchiveDeleteParallelism(), false);
+ FSUtils.deleteFilesParallelize(metaClient, manifestFilesToClean, context, config.getArchiveDeleteParallelism(),
+ false);
// delete the data files
- List<String> dataFilesToClean = Arrays.stream(LSMTimeline.listAllMetaFiles(metaClient))
+ List<String> dataFilesToClean = LSMTimeline.listAllMetaFiles(metaClient).stream()
.filter(fileStatus -> !filesToKeep.contains(fileStatus.getPath().getName()))
.map(fileStatus -> fileStatus.getPath().toString())
.collect(Collectors.toList());
- FSUtils.deleteFilesParallelize(metaClient, dataFilesToClean, context, config.getArchiveDeleteParallelism(), false);
+ FSUtils.deleteFilesParallelize(metaClient, dataFilesToClean, context,
+ config.getArchiveDeleteParallelism(), false);
}
}
private HoodieLSMTimelineManifest.LSMFileEntry getFileEntry(String fileName) throws IOException {
- long fileLen = metaClient.getFs().getFileStatus(new Path(metaClient.getArchivePath(), fileName)).getLen();
+ long fileLen = metaClient.getStorage().getPathInfo(
+ new StoragePath(metaClient.getArchivePath(), fileName)).getLength();
return HoodieLSMTimelineManifest.LSMFileEntry.getInstance(fileName, fileLen);
}
@@ -376,7 +379,7 @@ public class LSMTimelineWriter {
return this.writeConfig;
}
- private HoodieFileWriter openWriter(Path filePath) {
+ private HoodieFileWriter openWriter(StoragePath filePath) {
try {
return HoodieFileWriterFactory.getFileWriter("", filePath, metaClient.getHadoopConf(), getOrCreateWriterConfig(),
HoodieLSMTimelineInstant.getClassSchema(), taskContextSupplier, HoodieRecord.HoodieRecordType.AVRO);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/TransactionManager.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/TransactionManager.java
index b3e9abc7a3a..c02ed4a171c 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/TransactionManager.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/TransactionManager.java
@@ -22,6 +22,7 @@ import org.apache.hudi.client.transaction.lock.LockManager;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
@@ -41,8 +42,8 @@ public class TransactionManager implements Serializable {
protected Option<HoodieInstant> currentTxnOwnerInstant = Option.empty();
private Option<HoodieInstant> lastCompletedTxnOwnerInstant = Option.empty();
- public TransactionManager(HoodieWriteConfig config, FileSystem fs) {
- this(new LockManager(config, fs), config.isLockRequired());
+ public TransactionManager(HoodieWriteConfig config, HoodieStorage storage) {
+ this(new LockManager(config, (FileSystem) storage.getFileSystem()), config.isLockRequired());
}
protected TransactionManager(LockManager lockManager, boolean isLockRequired) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java
index 4309de5f556..24a0e1ce9f4 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java
@@ -20,7 +20,6 @@ package org.apache.hudi.client.utils;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.avro.model.HoodieArchivedMetaEntry;
-import org.apache.hudi.common.table.timeline.ActiveAction;
import org.apache.hudi.client.timeline.ActiveActionWithDetails;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodiePartitionMetadata;
@@ -29,6 +28,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.log.HoodieLogFormat;
import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock;
import org.apache.hudi.common.table.log.block.HoodieLogBlock;
+import org.apache.hudi.common.table.timeline.ActiveAction;
import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
@@ -36,11 +36,11 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.StoragePath;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,7 +49,6 @@ import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
@@ -140,13 +139,13 @@ public class LegacyArchivedMetaEntryReader {
private ClosableIterator<ActiveAction> loadInstants(HoodieArchivedTimeline.TimeRangeFilter filter) {
try {
// List all files
- FileStatus[] fsStatuses = metaClient.getFs().globStatus(
- new Path(metaClient.getArchivePath() + "/.commits_.archive*"));
+ List<StoragePathInfo> pathInfoList = metaClient.getStorage().globEntries(
+ new StoragePath(metaClient.getArchivePath() + "/.commits_.archive*"));
// Sort files by version suffix in reverse (implies reverse chronological order)
- Arrays.sort(fsStatuses, new ArchiveLogVersionComparator());
+ pathInfoList.sort(new ArchiveLogVersionComparator());
- ClosableIterator<HoodieRecord<IndexedRecord>> itr = getRecordIterator(fsStatuses);
+ ClosableIterator<HoodieRecord<IndexedRecord>> itr = getRecordIterator(pathInfoList);
return new ClosableIterator<ActiveAction>() {
private ActiveAction activeAction;
@@ -204,10 +203,11 @@ public class LegacyArchivedMetaEntryReader {
/**
* Returns the avro record iterator with given file statuses.
*/
- private ClosableIterator<HoodieRecord<IndexedRecord>> getRecordIterator(FileStatus[] fsStatuses) throws IOException {
+ private ClosableIterator<HoodieRecord<IndexedRecord>> getRecordIterator(
+ List<StoragePathInfo> pathInfoList) throws IOException {
return new ClosableIterator<HoodieRecord<IndexedRecord>>() {
- final Iterator<FileStatus> fsItr = Arrays.asList(fsStatuses).iterator();
+ final Iterator<StoragePathInfo> pathInfoIterator = pathInfoList.iterator();
HoodieLogFormat.Reader reader;
ClosableIterator<HoodieRecord<IndexedRecord>> recordItr;
@@ -240,13 +240,16 @@ public class LegacyArchivedMetaEntryReader {
reader = null;
}
// new reader
- while (fsItr.hasNext()) {
- FileStatus fs = fsItr.next();
+ while (pathInfoIterator.hasNext()) {
+ StoragePathInfo pathInfo = pathInfoIterator.next();
try {
- reader = HoodieLogFormat.newReader(metaClient.getFs(),
- new HoodieLogFile(fs.getPath()), HoodieArchivedMetaEntry.getClassSchema());
+ reader = HoodieLogFormat.newReader(
+ metaClient.getStorage(),
+ new HoodieLogFile(pathInfo.getPath()),
+ HoodieArchivedMetaEntry.getClassSchema());
} catch (IOException ioe) {
- throw new HoodieIOException("Error initializing the reader for archived log: " + fs.getPath(), ioe);
+ throw new HoodieIOException(
+ "Error initializing the reader for archived log: " + pathInfo.getPath(), ioe);
}
while (reader.hasNext()) {
HoodieLogBlock block = reader.next();
@@ -262,7 +265,7 @@ public class LegacyArchivedMetaEntryReader {
try {
reader.close();
} catch (IOException e) {
- throw new HoodieIOException("Failed to close log reader " + fs.getPath());
+ throw new HoodieIOException("Failed to close log reader " + pathInfo.getPath());
}
}
}
@@ -287,14 +290,15 @@ public class LegacyArchivedMetaEntryReader {
/**
* Sort files by reverse order of version suffix in file name.
*/
- public static class ArchiveLogVersionComparator implements Comparator<FileStatus>, Serializable {
+ public static class ArchiveLogVersionComparator
+ implements Comparator<StoragePathInfo>, Serializable {
@Override
- public int compare(FileStatus f1, FileStatus f2) {
+ public int compare(StoragePathInfo f1, StoragePathInfo f2) {
return Integer.compare(getArchivedFileSuffix(f2), getArchivedFileSuffix(f1));
}
}
- private static int getArchivedFileSuffix(FileStatus f) {
+ private static int getArchivedFileSuffix(StoragePathInfo f) {
try {
Matcher fileMatcher = ARCHIVE_FILE_PATTERN.matcher(f.getPath().getName());
if (fileMatcher.matches()) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
index fc065e0fe9a..6b5381b2fc0 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
@@ -47,11 +47,11 @@ import org.apache.hudi.io.HoodieMergedReadHandle;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.metadata.MetadataPartitionType;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -178,7 +178,8 @@ public class HoodieIndexUtils {
* @param candidateRecordKeys - Candidate keys to filter
* @return List of pairs of candidate keys and positions that are available in the file
*/
- public static List<Pair<String, Long>> filterKeysFromFile(Path filePath, List<String> candidateRecordKeys,
+ public static List<Pair<String, Long>> filterKeysFromFile(StoragePath filePath,
+ List<String> candidateRecordKeys,
Configuration configuration) throws HoodieIndexException {
checkArgument(FSUtils.isBaseFile(filePath));
List<Pair<String, Long>> foundRecordKeys = new ArrayList<>();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
index 0e47d0a688a..7a124d25ee9 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
@@ -29,10 +29,12 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieIndexException;
-import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,8 +108,8 @@ public class ConsistentBucketIndexUtils {
*/
public static Option<HoodieConsistentHashingMetadata> loadMetadata(HoodieTable table, String partition) {
HoodieTableMetaClient metaClient = table.getMetaClient();
- Path metadataPath = FSUtils.getPartitionPath(metaClient.getHashingMetadataPath(), partition);
- Path partitionPath = FSUtils.getPartitionPath(metaClient.getBasePathV2(), partition);
+ Path metadataPath = FSUtils.getPartitionPathInHadoopPath(metaClient.getHashingMetadataPath(), partition);
+ Path partitionPath = FSUtils.getPartitionPathInHadoopPath(metaClient.getBasePathV2().toString(), partition);
try {
Predicate<FileStatus> hashingMetaCommitFilePredicate = fileStatus -> {
String filename = fileStatus.getPath().getName();
@@ -117,7 +119,8 @@ public class ConsistentBucketIndexUtils {
String filename = fileStatus.getPath().getName();
return filename.contains(HASHING_METADATA_FILE_SUFFIX);
};
- final FileStatus[] metaFiles = metaClient.getFs().listStatus(metadataPath);
+ final FileStatus[] metaFiles =
+ ((FileSystem) metaClient.getStorage().getFileSystem()).listStatus(metadataPath);
final TreeSet<String> commitMetaTss = Arrays.stream(metaFiles).filter(hashingMetaCommitFilePredicate)
.map(commitFile -> HoodieConsistentHashingMetadata.getTimestampFromFile(commitFile.getPath().getName()))
.sorted()
@@ -182,10 +185,11 @@ public class ConsistentBucketIndexUtils {
* @return true if the metadata is saved successfully
*/
public static boolean saveMetadata(HoodieTable table, HoodieConsistentHashingMetadata metadata, boolean overwrite) {
- HoodieWrapperFileSystem fs = table.getMetaClient().getFs();
- Path dir = FSUtils.getPartitionPath(table.getMetaClient().getHashingMetadataPath(), metadata.getPartitionPath());
- Path fullPath = new Path(dir, metadata.getFilename());
- try (OutputStream out = fs.create(fullPath, overwrite)) {
+ HoodieStorage storage = table.getMetaClient().getStorage();
+ StoragePath dir = FSUtils.getPartitionPath(
+ table.getMetaClient().getHashingMetadataPath(), metadata.getPartitionPath());
+ StoragePath fullPath = new StoragePath(dir, metadata.getFilename());
+ try (OutputStream out = storage.create(fullPath, overwrite)) {
byte[] bytes = metadata.toBytes();
out.write(bytes);
out.close();
@@ -205,17 +209,18 @@ public class ConsistentBucketIndexUtils {
* @throws IOException
*/
private static void createCommitMarker(HoodieTable table, Path fileStatus, Path partitionPath) throws IOException {
- HoodieWrapperFileSystem fs = table.getMetaClient().getFs();
- Path fullPath = new Path(partitionPath, getTimestampFromFile(fileStatus.getName()) + HASHING_METADATA_COMMIT_FILE_SUFFIX);
- if (fs.exists(fullPath)) {
+ HoodieStorage storage = table.getMetaClient().getStorage();
+ StoragePath fullPath = new StoragePath(
+ partitionPath.toString(), getTimestampFromFile(fileStatus.getName()) + HASHING_METADATA_COMMIT_FILE_SUFFIX);
+ if (storage.exists(fullPath)) {
return;
}
//prevent exception from race condition. We are ok with the file being created in another thread, so we should
// check for the marker after catching the exception and we don't need to fail if the file exists
try {
- FileIOUtils.createFileInPath(fs, fullPath, Option.of(getUTF8Bytes(StringUtils.EMPTY_STRING)));
+ FileIOUtils.createFileInPath(storage, fullPath, Option.of(getUTF8Bytes(StringUtils.EMPTY_STRING)));
} catch (HoodieIOException e) {
- if (!fs.exists(fullPath)) {
+ if (!storage.exists(fullPath)) {
throw e;
}
LOG.warn("Failed to create marker but " + fullPath + " exists", e);
@@ -233,7 +238,8 @@ public class ConsistentBucketIndexUtils {
if (metaFile == null) {
return Option.empty();
}
- try (InputStream is = table.getMetaClient().getFs().open(metaFile.getPath())) {
+ try (InputStream is = table.getMetaClient().getStorage().open(
+ new StoragePath(metaFile.getPath().toUri()))) {
byte[] content = FileIOUtils.readAsByteArray(is);
return Option.of(HoodieConsistentHashingMetadata.fromBytes(content));
} catch (FileNotFoundException e) {
@@ -261,7 +267,7 @@ public class ConsistentBucketIndexUtils {
* @return true if hashing metadata file is latest else false
*/
private static boolean recommitMetadataFile(HoodieTable table, FileStatus metaFile, String partition) {
- Path partitionPath = FSUtils.getPartitionPath(table.getMetaClient().getBasePathV2(), partition);
+ Path partitionPath = new Path(FSUtils.getPartitionPath(table.getMetaClient().getBasePathV2(), partition).toUri());
String timestamp = getTimestampFromFile(metaFile.getPath().getName());
if (table.getPendingCommitTimeline().containsInstant(timestamp)) {
return false;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
index 1301f046ae2..0a18420be27 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
@@ -57,6 +57,7 @@ import org.apache.hudi.exception.HoodieAppendException;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
@@ -204,8 +205,9 @@ public class HoodieAppendHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
try {
// Save hoodie partition meta in the partition path
- HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(fs, instantTime,
- new Path(config.getBasePath()), FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
+ HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(storage, instantTime,
+ new StoragePath(config.getBasePath()),
+ FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
partitionMetadata.trySave(getPartitionId());
@@ -678,7 +680,7 @@ public class HoodieAppendHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
return new HoodieAvroDataBlock(records, shouldWriteRecordPositions, header, keyField);
case HFILE_DATA_BLOCK:
return new HoodieHFileDataBlock(
- records, header, writeConfig.getHFileCompressionAlgorithm(), new Path(writeConfig.getBasePath()),
+ records, header, writeConfig.getHFileCompressionAlgorithm(), new StoragePath(writeConfig.getBasePath()),
writeConfig.getBooleanOrDefault(HoodieReaderConfig.USE_NATIVE_HFILE_READER));
case PARQUET_DATA_BLOCK:
return new HoodieParquetDataBlock(
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
index 9c78c5aae7c..4b2b4c32be0 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
@@ -196,7 +196,7 @@ public class HoodieCDCLogger implements Closeable {
HoodieLogBlock block = new HoodieCDCDataBlock(records, cdcDataBlockHeader, keyField);
AppendResult result = cdcWriter.appendBlocks(Collections.singletonList(block));
- Path cdcAbsPath = result.logFile().getPath();
+ Path cdcAbsPath = new Path(result.logFile().getPath().toUri());
if (!cdcAbsPaths.contains(cdcAbsPath)) {
cdcAbsPaths.add(cdcAbsPath);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieConcatHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieConcatHandle.java
index 6eb482926c0..9555c22e7dc 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieConcatHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieConcatHandle.java
@@ -99,7 +99,8 @@ public class HoodieConcatHandle<T, I, K, O> extends HoodieMergeHandle<T, I, K, O
// NOTE: We're enforcing preservation of the record metadata to keep existing semantic
writeToFile(new HoodieKey(key, partitionPath), oldRecord, oldSchema, config.getPayloadConfig().getProps(), true);
} catch (IOException | RuntimeException e) {
- String errMsg = String.format("Failed to write old record into new file for key %s from old file %s to new file %s with writerSchema %s",
+ String errMsg = String.format(
+ "Failed to write old record into new file for key %s from old file %s to new file %s with writerSchema %s",
key, getOldFilePath(), newFilePath, writeSchemaWithMetaFields.toString(true));
LOG.debug("Old record is " + oldRecord);
throw new HoodieUpsertException(errMsg, e);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
index 0a0f3352069..6f3824ac34c 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
@@ -34,6 +34,7 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieInsertException;
import org.apache.hudi.io.storage.HoodieFileWriter;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
@@ -55,7 +56,7 @@ public class HoodieCreateHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
private static final Logger LOG = LoggerFactory.getLogger(HoodieCreateHandle.class);
protected HoodieFileWriter fileWriter;
- protected final Path path;
+ protected final StoragePath path;
protected long recordsWritten = 0;
protected long insertRecordsWritten = 0;
protected long recordsDeleted = 0;
@@ -95,13 +96,16 @@ public class HoodieCreateHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
this.path = makeNewPath(partitionPath);
try {
- HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(fs, instantTime,
- new Path(config.getBasePath()), FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
+ HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(storage, instantTime,
+ new StoragePath(config.getBasePath()),
+ FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
partitionMetadata.trySave(getPartitionId());
- createMarkerFile(partitionPath, FSUtils.makeBaseFileName(this.instantTime, this.writeToken, this.fileId, hoodieTable.getBaseFileExtension()));
- this.fileWriter = HoodieFileWriterFactory.getFileWriter(instantTime, path, hoodieTable.getHadoopConf(), config,
- writeSchemaWithMetaFields, this.taskContextSupplier, config.getRecordMerger().getRecordType());
+ createMarkerFile(partitionPath,
+ FSUtils.makeBaseFileName(this.instantTime, this.writeToken, this.fileId, hoodieTable.getBaseFileExtension()));
+ this.fileWriter =
+ HoodieFileWriterFactory.getFileWriter(instantTime, path, hoodieTable.getHadoopConf(), config,
+ writeSchemaWithMetaFields, this.taskContextSupplier, config.getRecordMerger().getRecordType());
} catch (IOException e) {
throw new HoodieInsertException("Failed to initialize HoodieStorageWriter for path " + path, e);
}
@@ -237,10 +241,10 @@ public class HoodieCreateHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
stat.setNumInserts(insertRecordsWritten);
stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT);
stat.setFileId(writeStatus.getFileId());
- stat.setPath(new Path(config.getBasePath()), path);
+ stat.setPath(new StoragePath(config.getBasePath()), path);
stat.setTotalWriteErrors(writeStatus.getTotalErrorRecords());
- long fileSize = FSUtils.getFileSize(fs, path);
+ long fileSize = FSUtils.getFileSize(fs, new Path(path.toUri()));
stat.setTotalWriteBytes(fileSize);
stat.setFileSizeInBytes(fileSize);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieIOHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieIOHandle.java
index d108a3f5de8..39400394048 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieIOHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieIOHandle.java
@@ -21,6 +21,7 @@ package org.apache.hudi.io;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.table.HoodieTable;
import org.apache.hadoop.fs.FileSystem;
@@ -29,6 +30,7 @@ public abstract class HoodieIOHandle<T, I, K, O> {
protected final String instantTime;
protected final HoodieWriteConfig config;
+ protected final HoodieStorage storage;
protected final FileSystem fs;
protected final HoodieTable<T, I, K, O> hoodieTable;
@@ -36,8 +38,9 @@ public abstract class HoodieIOHandle<T, I, K, O> {
this.instantTime = instantTime.orElse(StringUtils.EMPTY_STRING);
this.config = config;
this.hoodieTable = hoodieTable;
- this.fs = getFileSystem();
+ this.storage = getStorage();
+ this.fs = (FileSystem) storage.getFileSystem();
}
- public abstract FileSystem getFileSystem();
+ public abstract HoodieStorage getStorage();
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
index ab41a94c2a9..4f6f4eaa4d0 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
@@ -27,10 +27,9 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.keygen.BaseKeyGenerator;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hadoop.fs.Path;
-
import java.util.List;
import java.util.stream.Stream;
@@ -54,9 +53,9 @@ public class HoodieKeyLocationFetchHandle<T, I, K, O> extends HoodieReadHandle<T
private List<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieBaseFile baseFile) {
BaseFileUtils baseFileUtils = BaseFileUtils.getInstance(baseFile.getPath());
if (keyGeneratorOpt.isPresent()) {
- return baseFileUtils.fetchRecordKeysWithPositions(hoodieTable.getHadoopConf(), new Path(baseFile.getPath()), keyGeneratorOpt);
+ return baseFileUtils.fetchRecordKeysWithPositions(hoodieTable.getHadoopConf(), new StoragePath(baseFile.getPath()), keyGeneratorOpt);
} else {
- return baseFileUtils.fetchRecordKeysWithPositions(hoodieTable.getHadoopConf(), new Path(baseFile.getPath()));
+ return baseFileUtils.fetchRecordKeysWithPositions(hoodieTable.getHadoopConf(), new StoragePath(baseFile.getPath()));
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java
index db3be1c1842..cf344be59bd 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java
@@ -26,9 +26,9 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.index.HoodieIndexUtils;
import org.apache.hudi.io.storage.HoodieFileReader;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -102,7 +102,7 @@ public class HoodieKeyLookupHandle<T, I, K, O> extends HoodieReadHandle<T, I, K,
HoodieBaseFile baseFile = getLatestBaseFile();
List<Pair<String, Long>> matchingKeysAndPositions = HoodieIndexUtils.filterKeysFromFile(
- new Path(baseFile.getPath()), candidateRecordKeys, hoodieTable.getHadoopConf());
+ new StoragePath(baseFile.getPath()), candidateRecordKeys, hoodieTable.getHadoopConf());
LOG.info(
String.format("Total records (%d), bloom filter candidates (%d)/fp(%d), actual matches (%d)", totalKeysChecked,
candidateRecordKeys.size(), candidateRecordKeys.size() - matchingKeysAndPositions.size(), matchingKeysAndPositions.size()));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
index 22e6f50f4f7..c9f7aa4e6c8 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
@@ -47,6 +47,7 @@ import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.io.storage.HoodieFileWriter;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.keygen.BaseKeyGenerator;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
@@ -106,8 +107,8 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
protected HoodieFileWriter fileWriter;
protected boolean preserveMetadata = false;
- protected Path newFilePath;
- protected Path oldFilePath;
+ protected StoragePath newFilePath;
+ protected StoragePath oldFilePath;
protected long recordsWritten = 0;
protected long recordsDeleted = 0;
protected long updatedRecordsWritten = 0;
@@ -172,8 +173,9 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
String latestValidFilePath = baseFileToMerge.getFileName();
writeStatus.getStat().setPrevCommit(baseFileToMerge.getCommitTime());
- HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(fs, instantTime,
- new Path(config.getBasePath()), FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
+ HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(storage, instantTime,
+ new StoragePath(config.getBasePath()),
+ FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
partitionMetadata.trySave(getPartitionId());
@@ -206,7 +208,7 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
}
protected void setWriteStatusPath() {
- writeStatus.getStat().setPath(new Path(config.getBasePath()), newFilePath);
+ writeStatus.getStat().setPath(new StoragePath(config.getBasePath()), newFilePath);
}
protected void makeOldAndNewFilePaths(String partitionPath, String oldFileName, String newFileName) {
@@ -382,7 +384,7 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
writeToFile(new HoodieKey(key, partitionPath), oldRecord, oldSchema, props, true);
} catch (IOException | RuntimeException e) {
String errMsg = String.format("Failed to merge old record into new file for key %s from old file %s to new file %s with writerSchema %s",
- key, getOldFilePath(), newFilePath, writeSchemaWithMetaFields.toString(true));
+ key, getOldFilePath(), newFilePath, writeSchemaWithMetaFields.toString(true));
LOG.debug("Old record is " + oldRecord);
throw new HoodieUpsertException(errMsg, e);
}
@@ -436,7 +438,7 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
fileWriter.close();
fileWriter = null;
- long fileSizeInBytes = FSUtils.getFileSize(fs, newFilePath);
+ long fileSizeInBytes = FSUtils.getFileSize(fs, new Path(newFilePath.toUri()));
HoodieWriteStat stat = writeStatus.getStat();
stat.setTotalWriteBytes(fileSizeInBytes);
@@ -492,7 +494,7 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
return Collections.singletonList(statuses).iterator();
}
- public Path getOldFilePath() {
+ public StoragePath getOldFilePath() {
return oldFilePath;
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergedReadHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergedReadHandle.java
index 80be1264390..47acc9a17ae 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergedReadHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergedReadHandle.java
@@ -117,7 +117,7 @@ public class HoodieMergedReadHandle<T, I, K, O> extends HoodieReadHandle<T, I, K
List<String> logFilePaths = fileSlice.getLogFiles().sorted(HoodieLogFile.getLogFileComparator())
.map(l -> l.getPath().toString()).collect(toList());
return HoodieMergedLogRecordScanner.newBuilder()
- .withFileSystem(hoodieTable.getMetaClient().getFs())
+ .withStorage(storage)
.withBasePath(hoodieTable.getMetaClient().getBasePathV2().toString())
.withLogFilePaths(logFilePaths)
.withReaderSchema(readerSchema)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java
index 5b7985ba979..62b562ecd03 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieReadHandle.java
@@ -24,11 +24,10 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
import java.io.IOException;
/**
@@ -53,8 +52,8 @@ public abstract class HoodieReadHandle<T, I, K, O> extends HoodieIOHandle<T, I,
}
@Override
- public FileSystem getFileSystem() {
- return hoodieTable.getMetaClient().getFs();
+ public HoodieStorage getStorage() {
+ return hoodieTable.getMetaClient().getStorage();
}
public Pair<String, String> getPartitionPathFileIDPair() {
@@ -72,11 +71,11 @@ public abstract class HoodieReadHandle<T, I, K, O> extends HoodieIOHandle<T, I,
protected HoodieFileReader createNewFileReader() throws IOException {
return HoodieFileReaderFactory.getReaderFactory(this.config.getRecordMerger().getRecordType())
- .getFileReader(config, hoodieTable.getHadoopConf(), new Path(getLatestBaseFile().getPath()));
+ .getFileReader(config, hoodieTable.getHadoopConf(), new StoragePath(getLatestBaseFile().getPath()));
}
protected HoodieFileReader createNewFileReader(HoodieBaseFile hoodieBaseFile) throws IOException {
return HoodieFileReaderFactory.getReaderFactory(this.config.getRecordMerger().getRecordType())
- .getFileReader(config, hoodieTable.getHadoopConf(), new Path(hoodieBaseFile.getPath()));
+ .getFileReader(config, hoodieTable.getHadoopConf(), new StoragePath(hoodieBaseFile.getPath()));
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
index ab80629c941..aff264deba2 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
@@ -38,14 +38,14 @@ import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,26 +112,27 @@ public abstract class HoodieWriteHandle<T, I, K, O> extends HoodieIOHandle<T, I,
return FSUtils.makeWriteToken(getPartitionId(), getStageId(), getAttemptId());
}
- public Path makeNewPath(String partitionPath) {
- Path path = FSUtils.getPartitionPath(config.getBasePath(), partitionPath);
+ public StoragePath makeNewPath(String partitionPath) {
+ StoragePath path = FSUtils.getPartitionPath(config.getBasePath(), partitionPath);
try {
- if (!fs.exists(path)) {
- fs.mkdirs(path); // create a new partition as needed.
+ if (!storage.exists(path)) {
+ storage.createDirectory(path); // create a new partition as needed.
}
} catch (IOException e) {
throw new HoodieIOException("Failed to make dir " + path, e);
}
- return new Path(path.toString(), FSUtils.makeBaseFileName(instantTime, writeToken, fileId, hoodieTable.getBaseFileExtension()));
+ return new StoragePath(path.toString(),
+ FSUtils.makeBaseFileName(instantTime, writeToken, fileId, hoodieTable.getBaseFileExtension()));
}
/**
* Make new file path with given file name.
*/
- protected Path makeNewFilePath(String partitionPath, String fileName) {
- String relativePath = new Path((partitionPath.isEmpty() ? "" : partitionPath + "/")
+ protected StoragePath makeNewFilePath(String partitionPath, String fileName) {
+ String relativePath = new StoragePath((partitionPath.isEmpty() ? "" : partitionPath + "/")
+ fileName).toString();
- return new Path(config.getBasePath(), relativePath);
+ return new StoragePath(config.getBasePath(), relativePath);
}
/**
@@ -201,8 +202,8 @@ public abstract class HoodieWriteHandle<T, I, K, O> extends HoodieIOHandle<T, I,
public abstract IOType getIOType();
@Override
- public FileSystem getFileSystem() {
- return hoodieTable.getMetaClient().getFs();
+ public HoodieStorage getStorage() {
+ return hoodieTable.getMetaClient().getStorage();
}
public HoodieWriteConfig getConfig() {
@@ -245,7 +246,7 @@ public abstract class HoodieWriteHandle<T, I, K, O> extends HoodieIOHandle<T, I,
.withDeltaCommit(deltaCommitTime)
.withFileSize(0L)
.withSizeThreshold(config.getLogFileMaxSize())
- .withFs(fs)
+ .withStorage(storage)
.withRolloverLogWriteToken(writeToken)
.withLogWriteToken(writeToken)
.withFileCreationCallback(getLogCreationCallback())
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 7b175db53a5..d270a143a96 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -70,8 +70,12 @@ import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hudi.hadoop.fs.CachingPath;
+import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.hadoop.fs.SerializablePath;
import org.apache.hudi.index.HoodieIndexUtils;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.avro.Schema;
@@ -724,7 +728,12 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
List<DirectoryInfo> dirinfoList = new LinkedList<>();
List<String> allPartitionPaths = metadata.getAllPartitionPaths().stream()
.map(partitionPath -> dataWriteConfig.getBasePath() + "/" + partitionPath).collect(Collectors.toList());
- Map<String, FileStatus[]> partitionFileMap = metadata.getAllFilesInPartitions(allPartitionPaths);
+ Map<String, FileStatus[]> partitionFileMap = metadata.getAllFilesInPartitions(allPartitionPaths)
+ .entrySet()
+ .stream()
+ .collect(Collectors.toMap(e -> e.getKey(),
+ e -> e.getValue().stream().map(status -> HadoopFSUtils.convertToHadoopFileStatus(status))
+ .toArray(FileStatus[]::new)));
for (Map.Entry<String, FileStatus[]> entry : partitionFileMap.entrySet()) {
dirinfoList.add(new DirectoryInfo(entry.getKey(), entry.getValue(), initializationTime));
}
@@ -745,14 +754,15 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
int fileGroupCount) throws IOException {
String partitionName = HoodieIndexUtils.getPartitionNameFromPartitionType(metadataPartition, dataMetaClient, dataWriteConfig.getFunctionalIndexConfig().getIndexName());
// Remove all existing file groups or leftover files in the partition
- final Path partitionPath = new Path(metadataWriteConfig.getBasePath(), partitionName);
- FileSystem fs = metadataMetaClient.getFs();
+ final StoragePath partitionPath = new StoragePath(metadataWriteConfig.getBasePath(), partitionName);
+ HoodieStorage storage = metadataMetaClient.getStorage();
try {
- final FileStatus[] existingFiles = fs.listStatus(partitionPath);
- if (existingFiles.length > 0) {
+ final List<StoragePathInfo> existingFiles = storage.listDirectEntries(partitionPath);
+ if (existingFiles.size() > 0) {
LOG.warn("Deleting all existing files found in MDT partition {}", partitionName);
- fs.delete(partitionPath, true);
- ValidationUtils.checkState(!fs.exists(partitionPath), "Failed to delete MDT partition " + partitionName);
+ storage.deleteDirectory(partitionPath);
+ ValidationUtils.checkState(!storage.exists(partitionPath),
+ "Failed to delete MDT partition " + partitionName);
}
} catch (FileNotFoundException ignored) {
// If the partition did not exist yet, it will be created below
@@ -788,7 +798,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
.withLogVersion(HoodieLogFile.LOGFILE_BASE_VERSION)
.withFileSize(0L)
.withSizeThreshold(metadataWriteConfig.getLogFileMaxSize())
- .withFs(dataMetaClient.getFs())
+ .withStorage(dataMetaClient.getStorage())
.withRolloverLogWriteToken(HoodieLogFormat.DEFAULT_WRITE_TOKEN)
.withLogWriteToken(HoodieLogFormat.DEFAULT_WRITE_TOKEN)
.withFileExtension(HoodieLogFile.DELTA_EXTENSION).build()) {
@@ -806,7 +816,8 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
// first update table config
dataMetaClient.getTableConfig().setMetadataPartitionState(dataMetaClient, partitionPath, false);
LOG.warn("Deleting Metadata Table partition: {}", partitionPath);
- dataMetaClient.getFs().delete(new Path(metadataWriteConfig.getBasePath(), partitionPath), true);
+ dataMetaClient.getStorage()
+ .deleteDirectory(new StoragePath(metadataWriteConfig.getBasePath(), partitionPath));
// delete corresponding pending indexing instant file in the timeline
LOG.warn("Deleting pending indexing instant from the timeline for partition: {}", partitionPath);
deletePendingIndexingInstant(dataMetaClient, partitionPath);
@@ -1430,25 +1441,25 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
Map<String, List<String>> partitionFilesToDelete, List<String> partitionsToDelete) throws IOException {
for (String partition : metadata.fetchAllPartitionPaths()) {
- Path partitionPath = null;
+ StoragePath partitionPath = null;
if (StringUtils.isNullOrEmpty(partition) && !dataMetaClient.getTableConfig().isTablePartitioned()) {
- partitionPath = new Path(dataWriteConfig.getBasePath());
+ partitionPath = new StoragePath(dataWriteConfig.getBasePath());
} else {
- partitionPath = new Path(dataWriteConfig.getBasePath(), partition);
+ partitionPath = new StoragePath(dataWriteConfig.getBasePath(), partition);
}
final String partitionId = HoodieTableMetadataUtil.getPartitionIdentifierForFilesPartition(partition);
- FileStatus[] metadataFiles = metadata.getAllFilesInPartition(partitionPath);
+ List<StoragePathInfo> metadataFiles = metadata.getAllFilesInPartition(partitionPath);
if (!dirInfoMap.containsKey(partition)) {
// Entire partition has been deleted
partitionsToDelete.add(partitionId);
- if (metadataFiles != null && metadataFiles.length > 0) {
- partitionFilesToDelete.put(partitionId, Arrays.stream(metadataFiles).map(f -> f.getPath().getName()).collect(Collectors.toList()));
+ if (metadataFiles != null && metadataFiles.size() > 0) {
+ partitionFilesToDelete.put(partitionId, metadataFiles.stream().map(f -> f.getPath().getName()).collect(Collectors.toList()));
}
} else {
// Some files need to be cleaned and some to be added in the partition
Map<String, Long> fsFiles = dirInfoMap.get(partition).getFileNameToSizeMap();
- List<String> mdtFiles = Arrays.stream(metadataFiles).map(mdtFile -> mdtFile.getPath().getName()).collect(Collectors.toList());
- List<String> filesDeleted = Arrays.stream(metadataFiles).map(f -> f.getPath().getName())
+ List<String> mdtFiles = metadataFiles.stream().map(mdtFile -> mdtFile.getPath().getName()).collect(Collectors.toList());
+ List<String> filesDeleted = metadataFiles.stream().map(f -> f.getPath().getName())
.filter(n -> !fsFiles.containsKey(n)).collect(Collectors.toList());
Map<String, Long> filesToAdd = new HashMap<>();
// new files could be added to DT due to restore that just happened which may not be tracked in RestoreMetadata.
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
index aadb0d48685..85cfb4d278d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -75,6 +75,8 @@ import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.metadata.MetadataPartitionType;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.table.action.bootstrap.HoodieBootstrapWriteMetadata;
import org.apache.hudi.table.action.commit.HoodieMergeHelper;
@@ -85,7 +87,6 @@ import org.apache.hudi.table.storage.HoodieStorageLayout;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -706,11 +707,11 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
.flatMap(Collection::stream)
.collect(Collectors.toList()),
partitionFilePair -> {
- final FileSystem fileSystem = metaClient.getFs();
+ final HoodieStorage storage = metaClient.getStorage();
LOG.info("Deleting invalid data file=" + partitionFilePair);
// Delete
try {
- fileSystem.delete(new Path(partitionFilePair.getValue()), false);
+ storage.deleteFile(new StoragePath(partitionFilePair.getValue()));
} catch (IOException e) {
throw new HoodieIOException(e.getMessage(), e);
}
@@ -813,10 +814,11 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
}
private boolean waitForCondition(String partitionPath, Stream<Pair<String, String>> partitionFilePaths, FileVisibility visibility) {
- final FileSystem fileSystem = metaClient.getRawFs();
+ final HoodieStorage storage = metaClient.getRawHoodieStorage();
List<String> fileList = partitionFilePaths.map(Pair::getValue).collect(Collectors.toList());
try {
- getConsistencyGuard(fileSystem, config.getConsistencyGuardConfig()).waitTill(partitionPath, fileList, visibility);
+ getConsistencyGuard(storage, config.getConsistencyGuardConfig())
+ .waitTill(partitionPath, fileList, visibility);
} catch (IOException | TimeoutException ioe) {
LOG.error("Got exception while waiting for files to show up", ioe);
return false;
@@ -829,10 +831,13 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
* <p>
* Default consistencyGuard class is {@link OptimisticConsistencyGuard}.
*/
- public static ConsistencyGuard getConsistencyGuard(FileSystem fs, ConsistencyGuardConfig consistencyGuardConfig) throws IOException {
+ public static ConsistencyGuard getConsistencyGuard(HoodieStorage storage,
+ ConsistencyGuardConfig consistencyGuardConfig)
+ throws IOException {
try {
return consistencyGuardConfig.shouldEnableOptimisticConsistencyGuard()
- ? new OptimisticConsistencyGuard(fs, consistencyGuardConfig) : new FailSafeConsistencyGuard(fs, consistencyGuardConfig);
+ ? new OptimisticConsistencyGuard(storage, consistencyGuardConfig)
+ : new FailSafeConsistencyGuard(storage, consistencyGuardConfig);
} catch (Throwable e) {
throw new IOException("Could not load ConsistencyGuard ", e);
}
@@ -1055,10 +1060,10 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
if (clearAll && partitions.size() > 0) {
LOG.info("Clear hoodie.table.metadata.partitions in hoodie.properties");
metaClient.getTableConfig().setValue(TABLE_METADATA_PARTITIONS.key(), EMPTY_STRING);
- HoodieTableConfig.update(metaClient.getFs(), new Path(metaClient.getMetaPath()), metaClient.getTableConfig().getProps());
+ HoodieTableConfig.update(metaClient.getStorage(), new StoragePath(metaClient.getMetaPath()), metaClient.getTableConfig().getProps());
} else if (partitionType.isPresent() && partitions.remove(partitionType.get().getPartitionPath())) {
metaClient.getTableConfig().setValue(HoodieTableConfig.TABLE_METADATA_PARTITIONS.key(), String.join(",", partitions));
- HoodieTableConfig.update(metaClient.getFs(), new Path(metaClient.getMetaPath()), metaClient.getTableConfig().getProps());
+ HoodieTableConfig.update(metaClient.getStorage(), new StoragePath(metaClient.getMetaPath()), metaClient.getTableConfig().getProps());
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
index c7e294410e3..2d7cf46793d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
@@ -69,7 +69,7 @@ public class CleanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K,
public CleanActionExecutor(HoodieEngineContext context, HoodieWriteConfig config, HoodieTable<T, I, K, O> table, String instantTime, boolean skipLocking) {
super(context, config, table, instantTime);
- this.txnManager = new TransactionManager(config, table.getMetaClient().getFs());
+ this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
this.skipLocking = skipLocking;
}
@@ -91,7 +91,7 @@ public class CleanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K,
private static Stream<Pair<String, PartitionCleanStat>> deleteFilesFunc(Iterator<Pair<String, CleanFileInfo>> cleanFileInfo, HoodieTable table) {
Map<String, PartitionCleanStat> partitionCleanStatMap = new HashMap<>();
- FileSystem fs = table.getMetaClient().getFs();
+ FileSystem fs = (FileSystem) table.getMetaClient().getStorage().getFileSystem();
cleanFileInfo.forEachRemaining(partitionDelFileTuple -> {
String partitionPath = partitionDelFileTuple.getLeft();
@@ -152,7 +152,8 @@ public class CleanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K,
partitionsToBeDeleted.forEach(entry -> {
try {
if (!isNullOrEmpty(entry)) {
- deleteFileAndGetResult(table.getMetaClient().getFs(), table.getMetaClient().getBasePath() + "/" + entry);
+ deleteFileAndGetResult((FileSystem) table.getMetaClient().getStorage().getFileSystem(),
+ table.getMetaClient().getBasePath() + "/" + entry);
}
} catch (IOException e) {
LOG.warn("Partition deletion failed " + entry);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
index 753f8c8253d..9d1d1a0bc02 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
@@ -46,9 +46,9 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieSavepointException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -466,8 +466,9 @@ public class CleanPlanner<T, I, K, O> implements Serializable {
private boolean hasPendingFiles(String partitionPath) {
try {
HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(hoodieTable.getMetaClient(), hoodieTable.getActiveTimeline());
- Path fullPartitionPath = new Path(hoodieTable.getMetaClient().getBasePathV2(), partitionPath);
- fsView.addFilesToView(FSUtils.getAllDataFilesInPartition(hoodieTable.getMetaClient().getFs(), fullPartitionPath));
+ StoragePath fullPartitionPath = new StoragePath(hoodieTable.getMetaClient().getBasePathV2(), partitionPath);
+ fsView.addFilesToView(FSUtils.getAllDataFilesInPartition(
+ hoodieTable.getMetaClient().getStorage(), fullPartitionPath));
// use #getAllFileGroups(partitionPath) instead of #getAllFileGroups() to exclude the replaced file groups.
return fsView.getAllFileGroups(partitionPath).findAny().isPresent();
} catch (Exception ex) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
index 36f75b6a5b0..bae32945917 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
@@ -93,7 +93,8 @@ public abstract class BaseCommitActionExecutor<T, I, K, O, R>
this.extraMetadata = extraMetadata;
this.taskContextSupplier = context.getTaskContextSupplier();
// TODO : Remove this once we refactor and move out autoCommit method from here, since the TxnManager is held in {@link BaseHoodieWriteClient}.
- this.txnManagerOption = config.shouldAutoCommit() ? Option.of(new TransactionManager(config, table.getMetaClient().getFs())) : Option.empty();
+ this.txnManagerOption = config.shouldAutoCommit()
+ ? Option.of(new TransactionManager(config, table.getMetaClient().getStorage())) : Option.empty();
if (this.txnManagerOption.isPresent() && this.txnManagerOption.get().isLockRequired()) {
// these txn metadata are only needed for auto commit when optimistic concurrent control is also enabled
this.lastCompletedTxn = TransactionUtils.getLastCompletedTxnInstantAndMetadata(table.getMetaClient());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java
index 7fba0463292..340cff14dbd 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/HoodieMergeHelper.java
@@ -38,13 +38,13 @@ import org.apache.hudi.internal.schema.utils.SerDeHelper;
import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.util.ExecutorFactory;
import org.apache.avro.Schema;
import org.apache.avro.SchemaCompatibility;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -110,7 +110,8 @@ public class HoodieMergeHelper<T> extends BaseMergeHelper {
ClosableIterator<HoodieRecord> recordIterator;
Schema recordSchema;
if (baseFile.getBootstrapBaseFile().isPresent()) {
- Path bootstrapFilePath = new Path(baseFile.getBootstrapBaseFile().get().getPath());
+ StoragePath bootstrapFilePath =
+ new StoragePath(baseFile.getBootstrapBaseFile().get().getPath());
Configuration bootstrapFileConfig = new Configuration(table.getHadoopConf());
bootstrapFileReader = HoodieFileReaderFactory.getReaderFactory(recordType).newBootstrapFileReader(
baseFileReader,
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java
index 461794a8f75..9ede03b12cd 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java
@@ -43,13 +43,13 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.internal.schema.InternalSchema;
import org.apache.hudi.internal.schema.utils.SerDeHelper;
import org.apache.hudi.io.IOUtils;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieCompactionHandler;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.compact.strategy.CompactionStrategy;
import org.apache.avro.Schema;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -161,7 +161,7 @@ public abstract class HoodieCompactor<T, I, K, O> implements Serializable {
Option<InstantRange> instantRange,
TaskContextSupplier taskContextSupplier,
CompactionExecutionHelper executionHelper) throws IOException {
- FileSystem fs = metaClient.getFs();
+ HoodieStorage storage = metaClient.getStorage();
Schema readerSchema;
Option<InternalSchema> internalSchemaOption = Option.empty();
if (!StringUtils.isNullOrEmpty(config.getInternalSchema())) {
@@ -185,11 +185,12 @@ public abstract class HoodieCompactor<T, I, K, O> implements Serializable {
long maxMemoryPerCompaction = IOUtils.getMaxMemoryPerCompaction(taskContextSupplier, config);
LOG.info("MaxMemoryPerCompaction => " + maxMemoryPerCompaction);
- List<String> logFiles = operation.getDeltaFileNames().stream().map(
- p -> new Path(FSUtils.getPartitionPath(metaClient.getBasePath(), operation.getPartitionPath()), p).toString())
+ List<String> logFiles = operation.getDeltaFileNames().stream().map(p ->
+ new StoragePath(FSUtils.getPartitionPath(
+ metaClient.getBasePath(), operation.getPartitionPath()), p).toString())
.collect(toList());
HoodieMergedLogRecordScanner scanner = HoodieMergedLogRecordScanner.newBuilder()
- .withFileSystem(fs)
+ .withStorage(storage)
.withBasePath(metaClient.getBasePath())
.withLogFilePaths(logFiles)
.withReaderSchema(readerSchema)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java
index 7cc0e338bcf..a81ee663fa9 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/HoodieLogCompactionPlanGenerator.java
@@ -87,7 +87,7 @@ public class HoodieLogCompactionPlanGenerator<T extends HoodieRecordPayload, I,
+ fileSlice.getPartitionPath() + " eligible for log compaction.");
HoodieTableMetaClient metaClient = hoodieTable.getMetaClient();
HoodieUnMergedLogRecordScanner scanner = HoodieUnMergedLogRecordScanner.newBuilder()
- .withFileSystem(metaClient.getFs())
+ .withStorage(metaClient.getStorage())
.withBasePath(hoodieTable.getMetaClient().getBasePath())
.withLogFilePaths(fileSlice.getLogFiles()
.sorted(HoodieLogFile.getLogFileComparator())
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
index 753de122a9d..09a9b153db1 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
@@ -39,10 +39,10 @@ import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.metadata.HoodieMetadataMetrics;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.metadata.MetadataPartitionType;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.BaseActionExecutor;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -98,7 +98,7 @@ public class RunIndexActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I,
public RunIndexActionExecutor(HoodieEngineContext context, HoodieWriteConfig config, HoodieTable<T, I, K, O> table, String instantTime) {
super(context, config, table, instantTime);
- this.txnManager = new TransactionManager(config, table.getMetaClient().getFs());
+ this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
if (config.getMetadataConfig().enableMetrics()) {
this.metrics = Option.of(new HoodieMetadataMetrics(config.getMetricsConfig()));
} else {
@@ -213,7 +213,8 @@ public class RunIndexActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I,
});
table.getMetaClient().getTableConfig().setValue(TABLE_METADATA_PARTITIONS_INFLIGHT.key(), String.join(",", inflightPartitions));
table.getMetaClient().getTableConfig().setValue(TABLE_METADATA_PARTITIONS.key(), String.join(",", completedPartitions));
- HoodieTableConfig.update(table.getMetaClient().getFs(), new Path(table.getMetaClient().getMetaPath()), table.getMetaClient().getTableConfig().getProps());
+ HoodieTableConfig.update(table.getMetaClient().getStorage(),
+ new StoragePath(table.getMetaClient().getMetaPath()), table.getMetaClient().getTableConfig().getProps());
// delete metadata partition
requestedPartitions.forEach(partition -> {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java
index da85fc4d634..7c1551bb0e1 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/ScheduleIndexActionExecutor.java
@@ -81,7 +81,7 @@ public class ScheduleIndexActionExecutor<T, I, K, O> extends BaseActionExecutor<
super(context, config, table, instantTime);
this.partitionIndexTypes = partitionIndexTypes;
this.partitionPaths = partitionPaths;
- this.txnManager = new TransactionManager(config, table.getMetaClient().getFs());
+ this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/functional/BaseHoodieFunctionalIndexClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/functional/BaseHoodieFunctionalIndexClient.java
index f8a2bf64f6c..f767ca3dcb2 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/functional/BaseHoodieFunctionalIndexClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/functional/BaseHoodieFunctionalIndexClient.java
@@ -23,7 +23,6 @@ import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.storage.StoragePath;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,7 +53,8 @@ public abstract class BaseHoodieFunctionalIndexClient {
// update table config if necessary
if (!metaClient.getTableConfig().getProps().containsKey(HoodieTableConfig.INDEX_DEFINITION_PATH) || !metaClient.getTableConfig().getIndexDefinitionPath().isPresent()) {
metaClient.getTableConfig().setValue(HoodieTableConfig.INDEX_DEFINITION_PATH, indexMetaPath);
- HoodieTableConfig.update(metaClient.getFs(), new Path(metaClient.getMetaPath()), metaClient.getTableConfig().getProps());
+ HoodieTableConfig.update(metaClient.getStorage(),
+ new StoragePath(metaClient.getMetaPath()), metaClient.getTableConfig().getProps());
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
index e5c7aa40385..ad00fe052df 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
@@ -62,7 +62,7 @@ public abstract class BaseRestoreActionExecutor<T, I, K, O> extends BaseActionEx
String savepointToRestoreTimestamp) {
super(context, config, table, instantTime);
this.savepointToRestoreTimestamp = savepointToRestoreTimestamp;
- this.txnManager = new TransactionManager(config, table.getMetaClient().getFs());
+ this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
index 51159d3d5c3..8dd7afe2f84 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
@@ -37,11 +37,11 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieRollbackException;
import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.BaseActionExecutor;
import org.apache.hudi.table.marker.WriteMarkersFactory;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -89,7 +89,7 @@ public abstract class BaseRollbackActionExecutor<T, I, K, O> extends BaseActionE
this.deleteInstants = deleteInstants;
this.skipTimelinePublish = skipTimelinePublish;
this.skipLocking = skipLocking;
- this.txnManager = new TransactionManager(config, table.getMetaClient().getFs());
+ this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
}
/**
@@ -174,7 +174,7 @@ public abstract class BaseRollbackActionExecutor<T, I, K, O> extends BaseActionE
&& !commitTimeline.findInstantsAfter(instantTimeToRollback, Integer.MAX_VALUE).empty()) {
// check if remnants are from a previous LAZY rollback config, if yes, let out of order rollback continue
try {
- if (!HoodieHeartbeatClient.heartbeatExists(table.getMetaClient().getFs(),
+ if (!HoodieHeartbeatClient.heartbeatExists(table.getMetaClient().getStorage(),
config.getBasePath(), instantTimeToRollback)) {
throw new HoodieRollbackException(
"Found commits after time :" + instantTimeToRollback + ", please rollback greater commits first");
@@ -315,10 +315,10 @@ public abstract class BaseRollbackActionExecutor<T, I, K, O> extends BaseActionE
return;
}
- Path backupDir = new Path(config.getRollbackBackupDirectory());
+ StoragePath backupDir = new StoragePath(config.getRollbackBackupDirectory());
if (!backupDir.isAbsolute()) {
// Path specified is relative to the meta directory
- backupDir = new Path(table.getMetaClient().getMetaPath(), config.getRollbackBackupDirectory());
+ backupDir = new StoragePath(table.getMetaClient().getMetaPath(), config.getRollbackBackupDirectory());
}
// Determine the instants to back up
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java
index 2793b905e7c..66a877d5ee5 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java
@@ -33,8 +33,10 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieRollbackException;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -122,7 +124,7 @@ public class BaseRollbackHelper implements Serializable {
return partitionToRollbackStats.stream();
} else if (!rollbackRequest.getLogBlocksToBeDeleted().isEmpty()) {
HoodieLogFormat.Writer writer = null;
- final Path filePath;
+ final StoragePath filePath;
try {
String fileId = rollbackRequest.getFileId();
@@ -130,7 +132,7 @@ public class BaseRollbackHelper implements Serializable {
.onParentPath(FSUtils.getPartitionPath(metaClient.getBasePath(), rollbackRequest.getPartitionPath()))
.withFileId(fileId)
.withDeltaCommit(instantToRollback.getTimestamp())
- .withFs(metaClient.getFs())
+ .withStorage(metaClient.getStorage())
.withFileExtension(HoodieLogFile.DELTA_EXTENSION).build();
// generate metadata
@@ -157,17 +159,17 @@ public class BaseRollbackHelper implements Serializable {
// This step is intentionally done after writer is closed. Guarantees that
// getFileStatus would reflect correct stats and FileNotFoundException is not thrown in
// cloud-storage : HUDI-168
- Map<FileStatus, Long> filesToNumBlocksRollback = Collections.singletonMap(
- metaClient.getFs().getFileStatus(Objects.requireNonNull(filePath)),
+ Map<StoragePathInfo, Long> filesToNumBlocksRollback = Collections.singletonMap(
+ metaClient.getStorage().getPathInfo(Objects.requireNonNull(filePath)),
1L
);
return Collections.singletonList(
- Pair.of(rollbackRequest.getPartitionPath(),
- HoodieRollbackStat.newBuilder()
- .withPartitionPath(rollbackRequest.getPartitionPath())
- .withRollbackBlockAppendResults(filesToNumBlocksRollback)
- .build()))
+ Pair.of(rollbackRequest.getPartitionPath(),
+ HoodieRollbackStat.newBuilder()
+ .withPartitionPath(rollbackRequest.getPartitionPath())
+ .withRollbackBlockAppendResults(filesToNumBlocksRollback)
+ .build()))
.stream();
} else {
return Collections.singletonList(
@@ -192,7 +194,7 @@ public class BaseRollbackHelper implements Serializable {
boolean isDeleted = true;
if (doDelete) {
try {
- isDeleted = metaClient.getFs().delete(fullDeletePath);
+ isDeleted = ((FileSystem) metaClient.getStorage().getFileSystem()).delete(fullDeletePath);
} catch (FileNotFoundException e) {
// if first rollback attempt failed and retried again, chances that some files are already deleted.
isDeleted = true;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
index 27f152d0bbd..fd32b83587d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
@@ -32,7 +32,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieRollbackException;
-import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hadoop.fs.FileStatus;
@@ -109,7 +109,8 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
Supplier<FileStatus[]> filesToDelete = () -> {
try {
return fetchFilesFromInstant(instantToRollback, partitionPath, metaClient.getBasePath(), baseFileExtension,
- metaClient.getFs(), commitMetadataOptional, isCommitMetadataCompleted, tableType);
+ (FileSystem) metaClient.getStorage().getFileSystem(),
+ commitMetadataOptional, isCommitMetadataCompleted, tableType);
} catch (IOException e) {
throw new HoodieIOException("Fetching files to delete error", e);
}
@@ -140,7 +141,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
// have been written to the log files.
hoodieRollbackRequests.addAll(getHoodieRollbackRequests(partitionPath,
listBaseFilesToBeDeleted(instantToRollback.getTimestamp(), baseFileExtension, partitionPath,
- metaClient.getFs())));
+ (FileSystem) metaClient.getStorage().getFileSystem())));
} else {
// if this is part of a restore operation, we should rollback/delete entire file slice.
hoodieRollbackRequests.addAll(getHoodieRollbackRequests(partitionPath,
@@ -181,14 +182,17 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
PathFilter filter = (path) -> {
if (path.toString().contains(baseFileExtension)) {
String fileCommitTime = FSUtils.getCommitTime(path.getName());
- return HoodieTimeline.compareTimestamps(commit, HoodieTimeline.LESSER_THAN_OR_EQUALS, fileCommitTime);
+ return HoodieTimeline.compareTimestamps(commit, HoodieTimeline.LESSER_THAN_OR_EQUALS,
+ fileCommitTime);
} else if (FSUtils.isLogFile(path)) {
- String fileCommitTime = FSUtils.getDeltaCommitTimeFromLogPath(path);
+ String fileCommitTime = FSUtils.getDeltaCommitTimeFromLogPath(new StoragePath(path.toUri()));
return completionTimeQueryView.isSlicedAfterOrOn(commit, fileCommitTime);
}
return false;
};
- return metaClient.getFs().listStatus(FSUtils.getPartitionPath(config.getBasePath(), partitionPath), filter);
+ return ((FileSystem) metaClient.getStorage().getFileSystem())
+ .listStatus(FSUtils.getPartitionPathInHadoopPath(config.getBasePath(), partitionPath),
+ filter);
}
@NotNull
@@ -217,11 +221,11 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
}
return false;
};
- return fs.listStatus(FSUtils.getPartitionPath(config.getBasePath(), partitionPath), filter);
+ return fs.listStatus(FSUtils.getPartitionPathInHadoopPath(config.getBasePath(), partitionPath), filter);
}
private FileStatus[] fetchFilesFromInstant(HoodieInstant instantToRollback, String partitionPath, String basePath,
- String baseFileExtension, HoodieWrapperFileSystem fs,
+ String baseFileExtension, FileSystem fs,
Option<HoodieCommitMetadata> commitMetadataOptional,
Boolean isCommitMetadataCompleted,
HoodieTableType tableType) throws IOException {
@@ -236,7 +240,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
private FileStatus[] fetchFilesFromCommitMetadata(HoodieInstant instantToRollback, String partitionPath,
String basePath, HoodieCommitMetadata commitMetadata,
- String baseFileExtension, HoodieWrapperFileSystem fs)
+ String baseFileExtension, FileSystem fs)
throws IOException {
SerializablePathFilter pathFilter = getSerializablePathFilter(baseFileExtension, instantToRollback.getTimestamp());
Path[] filePaths = getFilesFromCommitMetadata(basePath, commitMetadata, partitionPath);
@@ -263,7 +267,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
* @throws IOException
*/
private FileStatus[] fetchFilesFromListFiles(HoodieInstant instantToRollback, String partitionPath, String basePath,
- String baseFileExtension, HoodieWrapperFileSystem fs)
+ String baseFileExtension, FileSystem fs)
throws IOException {
SerializablePathFilter pathFilter = getSerializablePathFilter(baseFileExtension, instantToRollback.getTimestamp());
Path[] filePaths = listFilesToBeDeleted(basePath, partitionPath);
@@ -278,7 +282,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
}
private static Path[] listFilesToBeDeleted(String basePath, String partitionPath) {
- return new Path[] {FSUtils.getPartitionPath(basePath, partitionPath)};
+ return new Path[] {FSUtils.getPartitionPathInHadoopPath(basePath, partitionPath)};
}
private static Path[] getFilesFromCommitMetadata(String basePath, HoodieCommitMetadata commitMetadata, String partitionPath) {
@@ -294,7 +298,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
return commit.equals(fileCommitTime);
} else if (FSUtils.isLogFile(path)) {
// Since the baseCommitTime is the only commit for new log files, it's okay here
- String fileCommitTime = FSUtils.getDeltaCommitTimeFromLogPath(path);
+ String fileCommitTime = FSUtils.getDeltaCommitTimeFromLogPath(new StoragePath(path.toUri()));
return commit.equals(fileCommitTime);
}
return false;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java
index c3ee30ed3f4..0fe9d46df5d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java
@@ -27,8 +27,8 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.storage.StoragePathInfo;
-import org.apache.hadoop.fs.FileStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -81,7 +81,7 @@ public class RollbackUtils {
checkArgument(stat1.getPartitionPath().equals(stat2.getPartitionPath()));
final List<String> successDeleteFiles = new ArrayList<>();
final List<String> failedDeleteFiles = new ArrayList<>();
- final Map<FileStatus, Long> commandBlocksCount = new HashMap<>();
+ final Map<StoragePathInfo, Long> commandBlocksCount = new HashMap<>();
final Map<String, Long> logFilesFromFailedCommit = new HashMap<>();
Option.ofNullable(stat1.getSuccessDeleteFiles()).ifPresent(successDeleteFiles::addAll);
Option.ofNullable(stat2.getSuccessDeleteFiles()).ifPresent(successDeleteFiles::addAll);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByCreationTimeStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByCreationTimeStrategy.java
index a350086f2dc..4abf292da1e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByCreationTimeStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByCreationTimeStrategy.java
@@ -41,7 +41,7 @@ public class KeepByCreationTimeStrategy extends KeepByTimeStrategy {
HoodieTableMetaClient metaClient = hoodieTable.getMetaClient();
return partitionPathsForTTL.stream().parallel().filter(part -> {
HoodiePartitionMetadata hoodiePartitionMetadata =
- new HoodiePartitionMetadata(metaClient.getFs(), FSUtils.getPartitionPath(metaClient.getBasePath(), part));
+ new HoodiePartitionMetadata(metaClient.getStorage(), FSUtils.getPartitionPath(metaClient.getBasePath(), part));
Option<String> instantOption = hoodiePartitionMetadata.readPartitionCreatedCommitTime();
if (instantOption.isPresent()) {
String instantTime = instantOption.get();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java
index f3326fe1d7a..726bc5f9069 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/DirectWriteMarkers.java
@@ -34,8 +34,12 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.table.HoodieTable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
@@ -59,15 +63,15 @@ import static org.apache.hudi.table.marker.ConflictDetectionUtils.getDefaultEarl
public class DirectWriteMarkers extends WriteMarkers {
private static final Logger LOG = LoggerFactory.getLogger(DirectWriteMarkers.class);
- private final transient FileSystem fs;
+ private final transient HoodieStorage storage;
- public DirectWriteMarkers(FileSystem fs, String basePath, String markerFolderPath, String instantTime) {
+ public DirectWriteMarkers(HoodieStorage storage, String basePath, String markerFolderPath, String instantTime) {
super(basePath, markerFolderPath, instantTime);
- this.fs = fs;
+ this.storage = storage;
}
public DirectWriteMarkers(HoodieTable table, String instantTime) {
- this(table.getMetaClient().getFs(),
+ this(table.getMetaClient().getStorage(),
table.getMetaClient().getBasePath(),
table.getMetaClient().getMarkerFolderPath(instantTime),
instantTime);
@@ -80,7 +84,7 @@ public class DirectWriteMarkers extends WriteMarkers {
* @param parallelism parallelism for deletion.
*/
public boolean deleteMarkerDir(HoodieEngineContext context, int parallelism) {
- return FSUtils.deleteDir(context, fs, markerDirPath, parallelism);
+ return FSUtils.deleteDir(context, storage, markerDirPath, parallelism);
}
/**
@@ -88,29 +92,29 @@ public class DirectWriteMarkers extends WriteMarkers {
* @throws IOException
*/
public boolean doesMarkerDirExist() throws IOException {
- return fs.exists(markerDirPath);
+ return storage.exists(markerDirPath);
}
@Override
public Set<String> createdAndMergedDataPaths(HoodieEngineContext context, int parallelism) throws IOException {
Set<String> dataFiles = new HashSet<>();
- FileStatus[] topLevelStatuses = fs.listStatus(markerDirPath);
+ List<StoragePathInfo> topLevelInfoList = storage.listDirectEntries(markerDirPath);
List<String> subDirectories = new ArrayList<>();
- for (FileStatus topLevelStatus: topLevelStatuses) {
- if (topLevelStatus.isFile()) {
- String pathStr = topLevelStatus.getPath().toString();
+ for (StoragePathInfo topLevelInfo: topLevelInfoList) {
+ if (topLevelInfo.isFile()) {
+ String pathStr = topLevelInfo.getPath().toString();
if (pathStr.contains(HoodieTableMetaClient.MARKER_EXTN) && !pathStr.endsWith(IOType.APPEND.name())) {
dataFiles.add(translateMarkerToDataPath(pathStr));
}
} else {
- subDirectories.add(topLevelStatus.getPath().toString());
+ subDirectories.add(topLevelInfo.getPath().toString());
}
}
if (subDirectories.size() > 0) {
parallelism = Math.min(subDirectories.size(), parallelism);
- SerializableConfiguration serializedConf = new SerializableConfiguration(fs.getConf());
+ SerializableConfiguration serializedConf = new SerializableConfiguration((Configuration) storage.getConf());
context.setJobStatus(this.getClass().getSimpleName(), "Obtaining marker files for all created, merged paths");
dataFiles.addAll(context.flatMap(subDirectories, directory -> {
Path path = new Path(directory);
@@ -140,7 +144,7 @@ public class DirectWriteMarkers extends WriteMarkers {
public Set<String> allMarkerFilePaths() throws IOException {
Set<String> markerFiles = new HashSet<>();
if (doesMarkerDirExist()) {
- FSUtils.processFiles(fs, markerDirPath.toString(), fileStatus -> {
+ FSUtils.processFiles(storage, markerDirPath.toString(), fileStatus -> {
markerFiles.add(MarkerUtils.stripMarkerFolderPrefix(fileStatus.getPath().toString(), basePath, instantTime));
return true;
}, false);
@@ -154,18 +158,18 @@ public class DirectWriteMarkers extends WriteMarkers {
* @param markerName the full marker name, e.g., "2021/08/13/file1.marker.CREATE"
* @return path of the marker file
*/
- public Option<Path> create(String markerName) {
- return create(new Path(markerDirPath, markerName), true);
+ public Option<StoragePath> create(String markerName) {
+ return create(new StoragePath(markerDirPath, markerName), true);
}
@Override
- protected Option<Path> create(String partitionPath, String fileName, IOType type, boolean checkIfExists) {
+ protected Option<StoragePath> create(String partitionPath, String fileName, IOType type, boolean checkIfExists) {
return create(getMarkerPath(partitionPath, fileName, type), checkIfExists);
}
@Override
- public Option<Path> createWithEarlyConflictDetection(String partitionPath, String dataFileName, IOType type, boolean checkIfExists,
- HoodieWriteConfig config, String fileId, HoodieActiveTimeline activeTimeline) {
+ public Option<StoragePath> createWithEarlyConflictDetection(String partitionPath, String dataFileName, IOType type, boolean checkIfExists,
+ HoodieWriteConfig config, String fileId, HoodieActiveTimeline activeTimeline) {
String strategyClassName = config.getEarlyConflictDetectionStrategyClassName();
if (!ReflectionUtils.isSubClass(strategyClassName, DirectMarkerBasedDetectionStrategy.class)) {
LOG.warn("Cannot use " + strategyClassName + " for direct markers.");
@@ -174,29 +178,31 @@ public class DirectWriteMarkers extends WriteMarkers {
}
DirectMarkerBasedDetectionStrategy strategy =
(DirectMarkerBasedDetectionStrategy) ReflectionUtils.loadClass(strategyClassName,
- fs, partitionPath, fileId, instantTime, activeTimeline, config);
+ new Class<?>[] {HoodieStorage.class, String.class, String.class, String.class,
+ HoodieActiveTimeline.class, HoodieWriteConfig.class},
+ storage, partitionPath, fileId, instantTime, activeTimeline, config);
strategy.detectAndResolveConflictIfNecessary();
return create(getMarkerPath(partitionPath, dataFileName, type), checkIfExists);
}
- private Option<Path> create(Path markerPath, boolean checkIfExists) {
+ private Option<StoragePath> create(StoragePath markerPath, boolean checkIfExists) {
HoodieTimer timer = HoodieTimer.start();
- Path dirPath = markerPath.getParent();
+ StoragePath dirPath = markerPath.getParent();
try {
- if (!fs.exists(dirPath)) {
- fs.mkdirs(dirPath); // create a new partition as needed.
+ if (!storage.exists(dirPath)) {
+ storage.createDirectory(dirPath); // create a new partition as needed.
}
} catch (IOException e) {
throw new HoodieIOException("Failed to make dir " + dirPath, e);
}
try {
- if (checkIfExists && fs.exists(markerPath)) {
+ if (checkIfExists && storage.exists(markerPath)) {
LOG.warn("Marker Path=" + markerPath + " already exists, cancel creation");
return Option.empty();
}
LOG.info("Creating Marker Path=" + markerPath);
- fs.create(markerPath, false).close();
+ storage.create(markerPath, false).close();
} catch (IOException e) {
throw new HoodieException("Failed to create marker file " + markerPath, e);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java
index d17c15efe40..af1819f4cda 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerBasedRollbackUtils.java
@@ -23,9 +23,9 @@ import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.table.marker.MarkerType;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,8 +63,8 @@ public class MarkerBasedRollbackUtils {
public static List<String> getAllMarkerPaths(HoodieTable table, HoodieEngineContext context,
String instant, int parallelism) throws IOException {
String markerDir = table.getMetaClient().getMarkerFolderPath(instant);
- FileSystem fileSystem = table.getMetaClient().getFs();
- Option<MarkerType> markerTypeOption = readMarkerType(fileSystem, markerDir);
+ HoodieStorage storage = table.getMetaClient().getStorage();
+ Option<MarkerType> markerTypeOption = readMarkerType(storage, markerDir);
// If there is no marker type file "MARKERS.type", first assume "DIRECT" markers are used.
// If not, then fallback to "TIMELINE_SERVER_BASED" markers.
@@ -75,22 +75,26 @@ public class MarkerBasedRollbackUtils {
} catch (IOException | IllegalArgumentException e) {
LOG.warn(String.format("%s not present and %s marker failed with error: %s. So, falling back to %s marker",
MARKER_TYPE_FILENAME, DIRECT, e.getMessage(), TIMELINE_SERVER_BASED));
- return getTimelineServerBasedMarkers(context, parallelism, markerDir, fileSystem);
+ return getTimelineServerBasedMarkers(context, parallelism, markerDir, storage);
}
}
switch (markerTypeOption.get()) {
case TIMELINE_SERVER_BASED:
// Reads all markers written by the timeline server
- return getTimelineServerBasedMarkers(context, parallelism, markerDir, fileSystem);
+ return getTimelineServerBasedMarkers(context, parallelism, markerDir, storage);
default:
throw new HoodieException(
"The marker type \"" + markerTypeOption.get().name() + "\" is not supported.");
}
}
- private static List<String> getTimelineServerBasedMarkers(HoodieEngineContext context, int parallelism, String markerDir, FileSystem fileSystem) {
- Map<String, Set<String>> markersMap = readTimelineServerBasedMarkersFromFileSystem(markerDir, fileSystem, context, parallelism);
+ private static List<String> getTimelineServerBasedMarkers(HoodieEngineContext context,
+ int parallelism,
+ String markerDir,
+ HoodieStorage storage) {
+ Map<String, Set<String>> markersMap =
+ readTimelineServerBasedMarkersFromFileSystem(markerDir, storage, context, parallelism);
return markersMap.values().stream()
.flatMap(Collection::stream)
.collect(Collectors.toList());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleDirectMarkerBasedDetectionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleDirectMarkerBasedDetectionStrategy.java
index 7c85a5a1805..8a0c5f4220f 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleDirectMarkerBasedDetectionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleDirectMarkerBasedDetectionStrategy.java
@@ -25,7 +25,7 @@ import org.apache.hudi.common.util.MarkerUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieEarlyConflictDetectionException;
import org.apache.hudi.exception.HoodieIOException;
-import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem;
+import org.apache.hudi.storage.HoodieStorage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,9 +50,9 @@ public class SimpleDirectMarkerBasedDetectionStrategy extends DirectMarkerBasedD
private final Set<HoodieInstant> completedCommitInstants;
private final long maxAllowableHeartbeatIntervalInMs;
- public SimpleDirectMarkerBasedDetectionStrategy(HoodieWrapperFileSystem fs, String partitionPath, String fileId, String instantTime,
+ public SimpleDirectMarkerBasedDetectionStrategy(HoodieStorage storage, String partitionPath, String fileId, String instantTime,
HoodieActiveTimeline activeTimeline, HoodieWriteConfig config) {
- super(fs, partitionPath, fileId, instantTime, activeTimeline, config);
+ super(storage, partitionPath, fileId, instantTime, activeTimeline, config);
this.basePath = config.getBasePath();
this.checkCommitConflict = config.earlyConflictDetectionCheckCommitConflict();
this.completedCommitInstants = new HashSet<>(activeTimeline.getCommitsTimeline().filterCompletedInstants().getInstants());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleTransactionDirectMarkerBasedDetectionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleTransactionDirectMarkerBasedDetectionStrategy.java
index f17f166656c..3d984ba781c 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleTransactionDirectMarkerBasedDetectionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleTransactionDirectMarkerBasedDetectionStrategy.java
@@ -22,8 +22,9 @@ import org.apache.hudi.client.transaction.DirectMarkerTransactionManager;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieEarlyConflictDetectionException;
-import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,15 +40,16 @@ public class SimpleTransactionDirectMarkerBasedDetectionStrategy
SimpleTransactionDirectMarkerBasedDetectionStrategy.class);
public SimpleTransactionDirectMarkerBasedDetectionStrategy(
- HoodieWrapperFileSystem fs, String partitionPath, String fileId, String instantTime,
+ HoodieStorage storage, String partitionPath, String fileId, String instantTime,
HoodieActiveTimeline activeTimeline, HoodieWriteConfig config) {
- super(fs, partitionPath, fileId, instantTime, activeTimeline, config);
+ super(storage, partitionPath, fileId, instantTime, activeTimeline, config);
}
@Override
public void detectAndResolveConflictIfNecessary() throws HoodieEarlyConflictDetectionException {
DirectMarkerTransactionManager txnManager =
- new DirectMarkerTransactionManager((HoodieWriteConfig) config, fs, partitionPath, fileId);
+ new DirectMarkerTransactionManager((HoodieWriteConfig) config,
+ (FileSystem) storage.getFileSystem(), partitionPath, fileId);
try {
// Need to do transaction before create marker file when using early conflict detection
txnManager.beginTransaction(instantTime);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java
index 50e4363ee20..c1cbc6f6573 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java
@@ -28,12 +28,12 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieEarlyConflictDetectionException;
import org.apache.hudi.exception.HoodieRemoteException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.util.HttpRequestClient;
import org.apache.hudi.util.HttpRequestClient.RequestMethod;
import com.fasterxml.jackson.core.type.TypeReference;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -128,7 +128,7 @@ public class TimelineServerBasedWriteMarkers extends WriteMarkers {
}
@Override
- protected Option<Path> create(String partitionPath, String fileName, IOType type, boolean checkIfExists) {
+ protected Option<StoragePath> create(String partitionPath, String fileName, IOType type, boolean checkIfExists) {
HoodieTimer timer = HoodieTimer.start();
String markerFileName = getMarkerFileName(fileName, type);
@@ -137,15 +137,15 @@ public class TimelineServerBasedWriteMarkers extends WriteMarkers {
LOG.info("[timeline-server-based] Created marker file " + partitionPath + "/" + markerFileName
+ " in " + timer.endTimer() + " ms");
if (success) {
- return Option.of(new Path(FSUtils.getPartitionPath(markerDirPath, partitionPath), markerFileName));
+ return Option.of(new StoragePath(FSUtils.getPartitionPath(markerDirPath, partitionPath), markerFileName));
} else {
return Option.empty();
}
}
@Override
- public Option<Path> createWithEarlyConflictDetection(String partitionPath, String fileName, IOType type, boolean checkIfExists,
- HoodieWriteConfig config, String fileId, HoodieActiveTimeline activeTimeline) {
+ public Option<StoragePath> createWithEarlyConflictDetection(String partitionPath, String fileName, IOType type, boolean checkIfExists,
+ HoodieWriteConfig config, String fileId, HoodieActiveTimeline activeTimeline) {
HoodieTimer timer = new HoodieTimer().startTimer();
String markerFileName = getMarkerFileName(fileName, type);
Map<String, String> paramsMap = getConfigMap(partitionPath, markerFileName, true);
@@ -156,7 +156,7 @@ public class TimelineServerBasedWriteMarkers extends WriteMarkers {
+ " in " + timer.endTimer() + " ms");
if (success) {
- return Option.of(new Path(FSUtils.getPartitionPath(markerDirPath, partitionPath), markerFileName));
+ return Option.of(new StoragePath(FSUtils.getPartitionPath(markerDirPath, partitionPath), markerFileName));
} else {
// this failed may due to early conflict detection, so we need to throw out.
throw new HoodieEarlyConflictDetectionException(new ConcurrentModificationException("Early conflict detected but cannot resolve conflicts for overlapping writes"));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
index 58ae9dcc758..58abde613a5 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
@@ -26,8 +26,8 @@ import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.storage.StoragePath;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,12 +46,12 @@ public abstract class WriteMarkers implements Serializable {
private static final Logger LOG = LoggerFactory.getLogger(WriteMarkers.class);
protected final String basePath;
- protected final transient Path markerDirPath;
+ protected final transient StoragePath markerDirPath;
protected final String instantTime;
public WriteMarkers(String basePath, String markerFolderPath, String instantTime) {
this.basePath = basePath;
- this.markerDirPath = new Path(markerFolderPath);
+ this.markerDirPath = new StoragePath(markerFolderPath);
this.instantTime = instantTime;
}
@@ -63,7 +63,7 @@ public abstract class WriteMarkers implements Serializable {
* @param type write IO type.
* @return the marker path.
*/
- public Option<Path> create(String partitionPath, String fileName, IOType type) {
+ public Option<StoragePath> create(String partitionPath, String fileName, IOType type) {
return create(partitionPath, fileName, type, false);
}
@@ -79,8 +79,8 @@ public abstract class WriteMarkers implements Serializable {
* @param activeTimeline Active timeline for the write operation.
* @return the marker path.
*/
- public Option<Path> create(String partitionPath, String fileName, IOType type, HoodieWriteConfig writeConfig,
- String fileId, HoodieActiveTimeline activeTimeline) {
+ public Option<StoragePath> create(String partitionPath, String fileName, IOType type, HoodieWriteConfig writeConfig,
+ String fileId, HoodieActiveTimeline activeTimeline) {
if (writeConfig.getWriteConcurrencyMode().isOptimisticConcurrencyControl() && writeConfig.isEarlyConflictDetectionEnable()) {
HoodieTimeline pendingCompactionTimeline = activeTimeline.filterPendingCompactionTimeline();
HoodieTimeline pendingReplaceTimeline = activeTimeline.filterPendingReplaceTimeline();
@@ -103,7 +103,7 @@ public abstract class WriteMarkers implements Serializable {
* @param type write IO type
* @return the marker path or empty option if already exists
*/
- public Option<Path> createIfNotExists(String partitionPath, String fileName, IOType type) {
+ public Option<StoragePath> createIfNotExists(String partitionPath, String fileName, IOType type) {
return create(partitionPath, fileName, type, true);
}
@@ -119,8 +119,8 @@ public abstract class WriteMarkers implements Serializable {
* @param activeTimeline Active timeline for the write operation.
* @return the marker path.
*/
- public Option<Path> createIfNotExists(String partitionPath, String fileName, IOType type, HoodieWriteConfig writeConfig,
- String fileId, HoodieActiveTimeline activeTimeline) {
+ public Option<StoragePath> createIfNotExists(String partitionPath, String fileName, IOType type, HoodieWriteConfig writeConfig,
+ String fileId, HoodieActiveTimeline activeTimeline) {
if (writeConfig.isEarlyConflictDetectionEnable()
&& writeConfig.getWriteConcurrencyMode().isOptimisticConcurrencyControl()) {
HoodieTimeline pendingCompactionTimeline = activeTimeline.filterPendingCompactionTimeline();
@@ -180,10 +180,10 @@ public abstract class WriteMarkers implements Serializable {
* @param type The IO type
* @return path of the marker file
*/
- protected Path getMarkerPath(String partitionPath, String fileName, IOType type) {
- Path path = FSUtils.getPartitionPath(markerDirPath, partitionPath);
+ protected StoragePath getMarkerPath(String partitionPath, String fileName, IOType type) {
+ StoragePath path = FSUtils.getPartitionPath(markerDirPath, partitionPath);
String markerFileName = getMarkerFileName(fileName, type);
- return new Path(path, markerFileName);
+ return new StoragePath(path, markerFileName);
}
/**
@@ -218,13 +218,13 @@ public abstract class WriteMarkers implements Serializable {
/**
* Creates a marker.
*
- * @param partitionPath partition path in the table
- * @param fileName file name
- * @param type write IO type
+ * @param partitionPath partition path in the table
+ * @param fileName file name
+ * @param type write IO type
* @param checkIfExists whether to check if the marker already exists
* @return the marker path or empty option if already exists and {@code checkIfExists} is true
*/
- abstract Option<Path> create(String partitionPath, String fileName, IOType type, boolean checkIfExists);
+ abstract Option<StoragePath> create(String partitionPath, String fileName, IOType type, boolean checkIfExists);
/**
* Creates a marker with early conflict detection for multi-writers. If conflict is detected,
@@ -239,6 +239,6 @@ public abstract class WriteMarkers implements Serializable {
* @param activeTimeline Active timeline for the write operation.
* @return the marker path or empty option if already exists and {@code checkIfExists} is true.
*/
- public abstract Option<Path> createWithEarlyConflictDetection(String partitionPath, String fileName, IOType type, boolean checkIfExists,
- HoodieWriteConfig config, String fileId, HoodieActiveTimeline activeTimeline);
+ public abstract Option<StoragePath> createWithEarlyConflictDetection(String partitionPath, String fileName, IOType type, boolean checkIfExists,
+ HoodieWriteConfig config, String fileId, HoodieActiveTimeline activeTimeline);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/repair/RepairUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/repair/RepairUtils.java
index 81ccb0a620a..672f358e6a4 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/repair/RepairUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/repair/RepairUtils.java
@@ -29,8 +29,8 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.storage.StoragePath;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import java.io.IOException;
@@ -55,19 +55,19 @@ public final class RepairUtils {
/**
* Tags the instant time of each base or log file from the input file paths.
*
- * @param basePath Base path of the table.
- * @param allPaths A {@link List} of file paths to tag.
+ * @param basePath Base path of the table.
+ * @param allPaths A {@link List} of file paths to tag.
* @return A {@link Map} of instant time in {@link String} to a {@link List} of relative file paths.
*/
public static Map<String, List<String>> tagInstantsOfBaseAndLogFiles(
- String basePath, List<Path> allPaths) {
+ String basePath, List<StoragePath> allPaths) {
// Instant time -> Set of base and log file paths
Map<String, List<String>> instantToFilesMap = new HashMap<>();
allPaths.forEach(path -> {
String instantTime = FSUtils.getCommitTime(path.getName());
instantToFilesMap.computeIfAbsent(instantTime, k -> new ArrayList<>());
instantToFilesMap.get(instantTime).add(
- FSUtils.getRelativePartitionPath(new Path(basePath), path));
+ FSUtils.getRelativePartitionPath(new StoragePath(basePath), path));
});
return instantToFilesMap;
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FiveToSixUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FiveToSixUpgradeHandler.java
index 69086b394bf..da006f435b1 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FiveToSixUpgradeHandler.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FiveToSixUpgradeHandler.java
@@ -26,9 +26,9 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieUpgradeDowngradeException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,10 +64,10 @@ public class FiveToSixUpgradeHandler implements UpgradeHandler {
compactionTimeline.getInstantsAsStream().forEach(
deleteInstant -> {
LOG.info("Deleting instant " + deleteInstant + " in auxiliary meta path " + metaClient.getMetaAuxiliaryPath());
- Path metaFile = new Path(metaClient.getMetaAuxiliaryPath(), deleteInstant.getFileName());
+ StoragePath metaFile = new StoragePath(metaClient.getMetaAuxiliaryPath(), deleteInstant.getFileName());
try {
- if (metaClient.getFs().exists(metaFile)) {
- metaClient.getFs().delete(metaFile, false);
+ if (metaClient.getStorage().exists(metaFile)) {
+ metaClient.getStorage().deleteFile(metaFile);
LOG.info("Deleted instant file in auxiliary meta path : " + metaFile);
}
} catch (IOException e) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SixToFiveDowngradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SixToFiveDowngradeHandler.java
index dc2b7498aef..b4c3f902132 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SixToFiveDowngradeHandler.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SixToFiveDowngradeHandler.java
@@ -39,6 +39,7 @@ import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.compact.CompactionTriggerStrategy;
import org.apache.hudi.table.action.compact.strategy.UnBoundedCompactionStrategy;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.util.HashMap;
@@ -115,7 +116,7 @@ public class SixToFiveDowngradeHandler implements DowngradeHandler {
.filter(instant -> instant.getState() == HoodieInstant.State.REQUESTED);
compactionTimeline.getInstantsAsStream().forEach(instant -> {
String fileName = instant.getFileName();
- FileIOUtils.copy(metaClient.getFs(),
+ FileIOUtils.copy((FileSystem) metaClient.getStorage().getFileSystem(),
new Path(metaClient.getMetaPath(), fileName),
new Path(metaClient.getMetaAuxiliaryPath(), fileName));
});
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToOneDowngradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToOneDowngradeHandler.java
index 34d671a7cf0..593a625ad87 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToOneDowngradeHandler.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToOneDowngradeHandler.java
@@ -29,14 +29,13 @@ import org.apache.hudi.common.util.MarkerUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.DirectWriteMarkers;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
@@ -93,44 +92,44 @@ public class TwoToOneDowngradeHandler implements DowngradeHandler {
HoodieEngineContext context,
int parallelism) throws IOException {
String markerDir = table.getMetaClient().getMarkerFolderPath(commitInstantTime);
- FileSystem fileSystem = HadoopFSUtils.getFs(markerDir, context.getHadoopConf().newCopy());
- Option<MarkerType> markerTypeOption = MarkerUtils.readMarkerType(fileSystem, markerDir);
+ HoodieStorage storage = HoodieStorageUtils.getStorage(markerDir, context.getHadoopConf().newCopy());
+ Option<MarkerType> markerTypeOption = MarkerUtils.readMarkerType(storage, markerDir);
if (markerTypeOption.isPresent()) {
switch (markerTypeOption.get()) {
case TIMELINE_SERVER_BASED:
// Reads all markers written by the timeline server
Map<String, Set<String>> markersMap =
MarkerUtils.readTimelineServerBasedMarkersFromFileSystem(
- markerDir, fileSystem, context, parallelism);
+ markerDir, storage, context, parallelism);
DirectWriteMarkers directWriteMarkers = new DirectWriteMarkers(table, commitInstantTime);
// Recreates the markers in the direct format
markersMap.values().stream().flatMap(Collection::stream)
.forEach(directWriteMarkers::create);
// Deletes marker type file
- MarkerUtils.deleteMarkerTypeFile(fileSystem, markerDir);
+ MarkerUtils.deleteMarkerTypeFile(storage, markerDir);
// Deletes timeline server based markers
- deleteTimelineBasedMarkerFiles(context, markerDir, fileSystem, parallelism);
+ deleteTimelineBasedMarkerFiles(context, markerDir, storage, parallelism);
break;
default:
throw new HoodieException("The marker type \"" + markerTypeOption.get().name()
+ "\" is not supported for rollback.");
}
} else {
- if (fileSystem.exists(new Path(markerDir))) {
+ if (storage.exists(new StoragePath(markerDir))) {
// In case of partial failures during downgrade, there is a chance that marker type file was deleted,
// but timeline server based marker files are left. So deletes them if any
- deleteTimelineBasedMarkerFiles(context, markerDir, fileSystem, parallelism);
+ deleteTimelineBasedMarkerFiles(context, markerDir, storage, parallelism);
}
}
}
private void deleteTimelineBasedMarkerFiles(HoodieEngineContext context, String markerDir,
- FileSystem fileSystem, int parallelism) throws IOException {
+ HoodieStorage storage, int parallelism) throws IOException {
// Deletes timeline based marker files if any.
- Predicate<FileStatus> prefixFilter = fileStatus ->
+ Predicate<StoragePathInfo> prefixFilter = fileStatus ->
fileStatus.getPath().getName().startsWith(MARKERS_FILENAME_PREFIX);
- FSUtils.parallelizeSubPathProcess(context, fileSystem, new Path(markerDir), parallelism,
- prefixFilter, pairOfSubPathAndConf ->
- FSUtils.deleteSubPath(pairOfSubPathAndConf.getKey(), pairOfSubPathAndConf.getValue(), false));
+ FSUtils.parallelizeSubPathProcess(context, storage, new StoragePath(markerDir), parallelism,
+ prefixFilter, pairOfSubPathAndConf ->
+ FSUtils.deleteSubPath(pairOfSubPathAndConf.getKey(), pairOfSubPathAndConf.getValue(), false));
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
index a19e067aae1..60a3d924a67 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
@@ -28,8 +28,8 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieUpgradeDowngradeException;
import org.apache.hudi.metadata.HoodieMetadataWriteUtils;
import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.storage.StoragePath;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,7 +49,6 @@ public class UpgradeDowngrade {
private HoodieTableMetaClient metaClient;
protected HoodieWriteConfig config;
protected HoodieEngineContext context;
- private transient FileSystem fs;
private Path updatedPropsFilePath;
private Path propsFilePath;
@@ -59,7 +58,6 @@ public class UpgradeDowngrade {
this.metaClient = metaClient;
this.config = config;
this.context = context;
- this.fs = metaClient.getFs();
this.updatedPropsFilePath = new Path(metaClient.getMetaPath(), HOODIE_UPDATED_PROPERTY_FILE);
this.propsFilePath = new Path(metaClient.getMetaPath(), HoodieTableConfig.HOODIE_PROPERTIES_FILE);
this.upgradeDowngradeHelper = upgradeDowngradeHelper;
@@ -113,7 +111,7 @@ public class UpgradeDowngrade {
String metadataTablePath = HoodieTableMetadata.getMetadataTableBasePath(
metaClient.getBasePathV2().toString());
try {
- if (metaClient.getFs().exists(new Path(metadataTablePath))) {
+ if (metaClient.getStorage().exists(new StoragePath(metadataTablePath))) {
HoodieTableMetaClient mdtMetaClient = HoodieTableMetaClient.builder()
.setConf(metaClient.getHadoopConf()).setBasePath(metadataTablePath).build();
HoodieWriteConfig mdtWriteConfig = HoodieMetadataWriteUtils.createMetadataWriteConfig(
@@ -159,7 +157,8 @@ public class UpgradeDowngrade {
}
metaClient.getTableConfig().setTableVersion(toVersion);
- HoodieTableConfig.update(metaClient.getFs(), new Path(metaClient.getMetaPath()), metaClient.getTableConfig().getProps());
+ HoodieTableConfig.update(metaClient.getStorage(),
+ new StoragePath(metaClient.getMetaPath()), metaClient.getTableConfig().getProps());
}
protected Map<ConfigProperty, String> upgrade(HoodieTableVersion fromVersion, HoodieTableVersion toVersion, String instantTime) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java
index 69e4c2b80fb..0037e3b301d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java
@@ -31,15 +31,14 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieRollbackException;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.rollback.BaseRollbackHelper;
import org.apache.hudi.table.action.rollback.ListingBasedRollbackStrategy;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -105,8 +104,8 @@ public class ZeroToOneUpgradeHandler implements UpgradeHandler {
// not feasible to differentiate MERGE from CREATE. hence creating with MERGE IOType for all base files.
writeMarkers.create(rollbackStat.getPartitionPath(), dataFileName, IOType.MERGE);
}
- for (FileStatus fileStatus : rollbackStat.getCommandBlocksCount().keySet()) {
- writeMarkers.create(rollbackStat.getPartitionPath(), getFileNameForMarkerFromLogFile(fileStatus.getPath().toString(), table), IOType.APPEND);
+ for (StoragePathInfo pathInfo : rollbackStat.getCommandBlocksCount().keySet()) {
+ writeMarkers.create(rollbackStat.getPartitionPath(), getFileNameForMarkerFromLogFile(pathInfo.getPath().toString(), table), IOType.APPEND);
}
}
}
@@ -133,7 +132,7 @@ public class ZeroToOneUpgradeHandler implements UpgradeHandler {
* @return the marker file name thus curated.
*/
private static String getFileNameForMarkerFromLogFile(String logFilePath, HoodieTable<?, ?, ?, ?> table) {
- Path logPath = new Path(table.getMetaClient().getBasePath(), logFilePath);
+ StoragePath logPath = new StoragePath(table.getMetaClient().getBasePath(), logFilePath);
String fileId = FSUtils.getFileIdFromLogPath(logPath);
String deltaInstant = FSUtils.getDeltaCommitTimeFromLogPath(logPath);
String writeToken = FSUtils.getWriteTokenFromLogPath(logPath);
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
index 0b79dc3ee3c..9b61637136c 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
@@ -18,10 +18,6 @@
package org.apache.hudi.avro;
-import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.DummyTaskContextSupplier;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
@@ -31,6 +27,11 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ParquetUtils;
import org.apache.hudi.io.storage.HoodieAvroParquetWriter;
import org.apache.hudi.io.storage.HoodieParquetConfig;
+import org.apache.hudi.storage.StoragePath;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
@@ -70,10 +71,10 @@ public class TestHoodieAvroParquetWriter {
new HoodieParquetConfig(writeSupport, CompressionCodecName.GZIP, ParquetWriter.DEFAULT_BLOCK_SIZE,
ParquetWriter.DEFAULT_PAGE_SIZE, 1024 * 1024 * 1024, hadoopConf, 0.1, true);
- Path filePath = new Path(tmpDir.resolve("test.parquet").toAbsolutePath().toString());
+ StoragePath filePath = new StoragePath(tmpDir.resolve("test.parquet").toAbsolutePath().toString());
try (HoodieAvroParquetWriter writer =
- new HoodieAvroParquetWriter(filePath, parquetConfig, "001", new DummyTaskContextSupplier(), true)) {
+ new HoodieAvroParquetWriter(filePath, parquetConfig, "001", new DummyTaskContextSupplier(), true)) {
for (GenericRecord record : records) {
writer.writeAvro((String) record.get("_row_key"), record);
}
@@ -92,7 +93,8 @@ public class TestHoodieAvroParquetWriter {
String minKey = recordKeys.stream().min(Comparator.naturalOrder()).get();
String maxKey = recordKeys.stream().max(Comparator.naturalOrder()).get();
- FileMetaData parquetMetadata = ParquetUtils.readMetadata(hadoopConf, filePath).getFileMetaData();
+ FileMetaData parquetMetadata = ParquetUtils.readMetadata(
+ hadoopConf, filePath).getFileMetaData();
Map<String, String> extraMetadata = parquetMetadata.getKeyValueMetaData();
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java
index a877d6bfc23..85e7e484312 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java
@@ -18,13 +18,15 @@
package org.apache.hudi.client.heartbeat;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.StoragePath;
+
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
+import java.util.List;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.awaitility.Awaitility.await;
@@ -46,17 +48,21 @@ public class TestHoodieHeartbeatClient extends HoodieCommonTestHarness {
@Test
public void testStartHeartbeat() throws IOException {
HoodieHeartbeatClient hoodieHeartbeatClient =
- new HoodieHeartbeatClient(metaClient.getFs(), metaClient.getBasePath(), heartBeatInterval, numTolerableMisses);
+ new HoodieHeartbeatClient(metaClient.getStorage(), metaClient.getBasePath(),
+ heartBeatInterval,
+ numTolerableMisses);
hoodieHeartbeatClient.start(instantTime1);
- FileStatus [] fs = metaClient.getFs().listStatus(new Path(hoodieHeartbeatClient.getHeartbeatFolderPath()));
- assertTrue(fs.length == 1);
- assertTrue(fs[0].getPath().toString().contains(instantTime1));
+ List<StoragePathInfo> listFiles = metaClient.getStorage().listDirectEntries(
+ new StoragePath(hoodieHeartbeatClient.getHeartbeatFolderPath()));
+ assertTrue(listFiles.size() == 1);
+ assertTrue(listFiles.get(0).getPath().toString().contains(instantTime1));
}
@Test
public void testStopHeartbeat() {
HoodieHeartbeatClient hoodieHeartbeatClient =
- new HoodieHeartbeatClient(metaClient.getFs(), metaClient.getBasePath(), heartBeatInterval, numTolerableMisses);
+ new HoodieHeartbeatClient(metaClient.getStorage(), metaClient.getBasePath(),
+ heartBeatInterval, numTolerableMisses);
hoodieHeartbeatClient.start(instantTime1);
hoodieHeartbeatClient.stop(instantTime1);
await().atMost(5, SECONDS).until(() -> hoodieHeartbeatClient.getHeartbeat(instantTime1).getNumHeartbeats() > 0);
@@ -67,7 +73,8 @@ public class TestHoodieHeartbeatClient extends HoodieCommonTestHarness {
@Test
public void testIsHeartbeatExpired() throws IOException {
HoodieHeartbeatClient hoodieHeartbeatClient =
- new HoodieHeartbeatClient(metaClient.getFs(), metaClient.getBasePath(), heartBeatInterval, numTolerableMisses);
+ new HoodieHeartbeatClient(metaClient.getStorage(), metaClient.getBasePath(),
+ heartBeatInterval, numTolerableMisses);
hoodieHeartbeatClient.start(instantTime1);
hoodieHeartbeatClient.stop(instantTime1);
assertFalse(hoodieHeartbeatClient.isHeartbeatExpired(instantTime1));
@@ -77,7 +84,8 @@ public class TestHoodieHeartbeatClient extends HoodieCommonTestHarness {
public void testNumHeartbeatsGenerated() {
Long heartBeatInterval = 5000L;
HoodieHeartbeatClient hoodieHeartbeatClient =
- new HoodieHeartbeatClient(metaClient.getFs(), metaClient.getBasePath(), heartBeatInterval, numTolerableMisses);
+ new HoodieHeartbeatClient(metaClient.getStorage(), metaClient.getBasePath(),
+ heartBeatInterval, numTolerableMisses);
hoodieHeartbeatClient.start("100");
await().atMost(5, SECONDS).until(() -> hoodieHeartbeatClient.getHeartbeat(instantTime1).getNumHeartbeats() >= 1);
}
@@ -85,16 +93,19 @@ public class TestHoodieHeartbeatClient extends HoodieCommonTestHarness {
@Test
public void testDeleteWrongHeartbeat() throws IOException {
HoodieHeartbeatClient hoodieHeartbeatClient =
- new HoodieHeartbeatClient(metaClient.getFs(), metaClient.getBasePath(), heartBeatInterval, numTolerableMisses);
+ new HoodieHeartbeatClient(metaClient.getStorage(), metaClient.getBasePath(),
+ heartBeatInterval, numTolerableMisses);
hoodieHeartbeatClient.start(instantTime1);
hoodieHeartbeatClient.stop(instantTime1);
- assertFalse(HeartbeatUtils.deleteHeartbeatFile(metaClient.getFs(), basePath, instantTime2));
+ assertFalse(
+ HeartbeatUtils.deleteHeartbeatFile(metaClient.getStorage(), basePath, instantTime2));
}
@Test
public void testStopHeartbeatTimers() throws IOException {
HoodieHeartbeatClient hoodieHeartbeatClient =
- new HoodieHeartbeatClient(metaClient.getFs(), metaClient.getBasePath(), heartBeatInterval, numTolerableMisses);
+ new HoodieHeartbeatClient(metaClient.getStorage(), metaClient.getBasePath(),
+ heartBeatInterval, numTolerableMisses);
hoodieHeartbeatClient.start(instantTime1);
hoodieHeartbeatClient.stopHeartbeatTimers();
assertFalse(hoodieHeartbeatClient.isHeartbeatExpired(instantTime1));
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java
index 35a7a7be0b4..2581e1cc7be 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java
@@ -143,8 +143,12 @@ public class TestCompletionTimeQueryView {
writer.write(activeActions.subList(2, 4), Option.empty(), Option.empty());
writer.write(activeActions.subList(4, 6), Option.empty(), Option.empty());
// reconcile the active timeline
- instants.subList(0, 3 * 6).forEach(instant -> HoodieActiveTimeline.deleteInstantFile(metaClient.getFs(), metaClient.getMetaPath(), instant));
- ValidationUtils.checkState(metaClient.reloadActiveTimeline().filterCompletedInstants().countInstants() == 4, "should archive 6 instants with 4 as active");
+ instants.subList(0, 3 * 6).forEach(
+ instant -> HoodieActiveTimeline.deleteInstantFile(metaClient.getStorage(),
+ metaClient.getMetaPath(), instant));
+ ValidationUtils.checkState(
+ metaClient.reloadActiveTimeline().filterCompletedInstants().countInstants() == 4,
+ "should archive 6 instants with 4 as active");
}
@SuppressWarnings("rawtypes")
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestLockManager.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestLockManager.java
index 1b4c08c5329..398ce60f811 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestLockManager.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestLockManager.java
@@ -28,6 +28,7 @@ import org.apache.hudi.config.HoodieLockConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.fs.FileSystem;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -78,7 +79,8 @@ public class TestLockManager extends HoodieCommonTestHarness {
@ValueSource(booleans = {true, false})
void testLockAndUnlock(boolean multiWriter) {
HoodieWriteConfig writeConfig = multiWriter ? getMultiWriterWriteConfig() : getSingleWriterWriteConfig();
- LockManager lockManager = new LockManager(writeConfig, this.metaClient.getFs());
+ LockManager lockManager = new LockManager(writeConfig,
+ (FileSystem) this.metaClient.getStorage().getFileSystem());
LockManager mockLockManager = Mockito.spy(lockManager);
assertDoesNotThrow(() -> {
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestTransactionManager.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestTransactionManager.java
index c0fb8de8691..bf11ace20ce 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestTransactionManager.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestTransactionManager.java
@@ -32,16 +32,17 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.exception.HoodieLockException;
import org.apache.hudi.metrics.MetricsReporterType;
+
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
import java.io.IOException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.junit.jupiter.api.TestInfo;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -55,7 +56,7 @@ public class TestTransactionManager extends HoodieCommonTestHarness {
initPath();
initMetaClient();
this.writeConfig = getWriteConfig(testInfo.getTags().contains("useLockProviderWithRuntimeError"));
- this.transactionManager = new TransactionManager(this.writeConfig, this.metaClient.getFs());
+ this.transactionManager = new TransactionManager(this.writeConfig, this.metaClient.getStorage());
}
private HoodieWriteConfig getWriteConfig(boolean useLockProviderWithRuntimeError) {
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java
index 91831adb36c..cbd6f93f1c5 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestLegacyArchivedMetaEntryReader.java
@@ -44,7 +44,6 @@ import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
@@ -102,9 +101,10 @@ public class TestLegacyArchivedMetaEntryReader {
private HoodieLogFormat.Writer openWriter(HoodieTableMetaClient metaClient) {
try {
- return HoodieLogFormat.newWriterBuilder().onParentPath(new Path(metaClient.getArchivePath()))
+ return HoodieLogFormat.newWriterBuilder()
+ .onParentPath(new StoragePath(metaClient.getArchivePath()))
.withFileId("commits").withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION)
- .withFs(metaClient.getFs()).withDeltaCommit("").build();
+ .withStorage(metaClient.getStorage()).withDeltaCommit("").build();
} catch (IOException e) {
throw new HoodieException("Unable to initialize HoodieLogFormat writer", e);
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/testutils/HoodieMetadataTestTable.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/testutils/HoodieMetadataTestTable.java
index 5024a9f59d2..ae6de2f680d 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/testutils/HoodieMetadataTestTable.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/testutils/HoodieMetadataTestTable.java
@@ -33,8 +33,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
-
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.hudi.storage.HoodieStorage;
import java.io.IOException;
import java.util.List;
@@ -50,9 +49,11 @@ public class HoodieMetadataTestTable extends HoodieTestTable {
private final HoodieTableMetadataWriter writer;
- protected HoodieMetadataTestTable(String basePath, FileSystem fs, HoodieTableMetaClient metaClient, HoodieTableMetadataWriter writer,
+ protected HoodieMetadataTestTable(String basePath, HoodieStorage storage,
+ HoodieTableMetaClient metaClient,
+ HoodieTableMetadataWriter writer,
Option<HoodieEngineContext> context) {
- super(basePath, fs, metaClient, context);
+ super(basePath, storage, metaClient, context);
this.writer = writer;
}
@@ -60,9 +61,13 @@ public class HoodieMetadataTestTable extends HoodieTestTable {
return HoodieMetadataTestTable.of(metaClient, null, Option.empty());
}
- public static HoodieTestTable of(HoodieTableMetaClient metaClient, HoodieTableMetadataWriter writer, Option<HoodieEngineContext> context) {
+ public static HoodieTestTable of(HoodieTableMetaClient metaClient,
+ HoodieTableMetadataWriter writer,
+ Option<HoodieEngineContext> context) {
testTableState = HoodieTestTableState.of();
- return new HoodieMetadataTestTable(metaClient.getBasePath(), metaClient.getRawFs(), metaClient, writer, context);
+ return new HoodieMetadataTestTable(metaClient.getBasePath(), metaClient.getRawHoodieStorage(),
+ metaClient,
+ writer, context);
}
/**
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/marker/TestWriteMarkersFactory.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/marker/TestWriteMarkersFactory.java
index d78b8830682..e369e9694ad 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/marker/TestWriteMarkersFactory.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/marker/TestWriteMarkersFactory.java
@@ -27,6 +27,7 @@ import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.table.HoodieTable;
import org.apache.hadoop.conf.Configuration;
@@ -48,6 +49,7 @@ public class TestWriteMarkersFactory extends HoodieCommonTestHarness {
private static final String HDFS_BASE_PATH = "hdfs://localhost/dir";
private final HoodieWriteConfig writeConfig = Mockito.mock(HoodieWriteConfig.class);
private final HoodieTableMetaClient metaClient = Mockito.mock(HoodieTableMetaClient.class);
+ private final HoodieStorage storage = Mockito.mock(HoodieStorage.class);
private final HoodieWrapperFileSystem fileSystem = Mockito.mock(HoodieWrapperFileSystem.class);
private final HoodieEngineContext context = Mockito.mock(HoodieEngineContext.class);
private final HoodieTable table = Mockito.mock(HoodieTable.class);
@@ -103,11 +105,13 @@ public class TestWriteMarkersFactory extends HoodieCommonTestHarness {
Mockito.when(writeConfig.isEmbeddedTimelineServerEnabled())
.thenReturn(isTimelineServerEnabled);
Mockito.when(table.getMetaClient()).thenReturn(metaClient);
- Mockito.when(metaClient.getFs()).thenReturn(fileSystem);
+ Mockito.when(metaClient.getStorage()).thenReturn(storage);
+ Mockito.when(storage.getFileSystem()).thenReturn(fileSystem);
Mockito.when(metaClient.getBasePath()).thenReturn(basePath);
Mockito.when(metaClient.getMarkerFolderPath(any())).thenReturn(basePath + ".hoodie/.temp");
Mockito.when(table.getContext()).thenReturn(context);
- Mockito.when(context.getHadoopConf()).thenReturn(new SerializableConfiguration(new Configuration()));
+ Mockito.when(context.getHadoopConf())
+ .thenReturn(new SerializableConfiguration(new Configuration()));
Mockito.when(writeConfig.getViewStorageConfig())
.thenReturn(FileSystemViewStorageConfig.newBuilder().build());
assertEquals(expectedWriteMarkersClass,
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/repair/TestRepairUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/repair/TestRepairUtils.java
index 4f8fb1dba33..7dfdba5ff6d 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/repair/TestRepairUtils.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/repair/TestRepairUtils.java
@@ -29,6 +29,7 @@ import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.BeforeAll;
@@ -77,7 +78,7 @@ public class TestRepairUtils {
@Test
public void testTagInstantsOfBaseAndLogFiles() {
Map<String, List<String>> expectedResult = new HashMap<>();
- List<Path> inputPathList = new ArrayList<>();
+ List<StoragePath> inputPathList = new ArrayList<>();
for (Map.Entry<String, List<Pair<String, String>>> entry : BASE_FILE_INFO.entrySet()) {
String instantTime = entry.getKey();
@@ -85,15 +86,15 @@ public class TestRepairUtils {
.map(e -> {
String partitionPath = e.getKey();
String fileId = e.getValue();
- return new Path(
- new Path(partitionPath), getBaseFilename(instantTime, fileId)).toString();
+ return new StoragePath(
+ new StoragePath(partitionPath), getBaseFilename(instantTime, fileId)).toString();
})
.collect(Collectors.toList());
List<String> expectedList = expectedResult.computeIfAbsent(
instantTime, k -> new ArrayList<>());
expectedList.addAll(fileNameList);
inputPathList.addAll(fileNameList.stream()
- .map(path -> new Path(basePath, path)).collect(Collectors.toList()));
+ .map(path -> new StoragePath(basePath, path)).collect(Collectors.toList()));
}
for (Map.Entry<String, List<Pair<String, String>>> entry : LOG_FILE_INFO.entrySet()) {
@@ -102,15 +103,15 @@ public class TestRepairUtils {
.map(e -> {
String partitionPath = e.getKey();
String fileId = e.getValue();
- return new Path(
- new Path(partitionPath), getLogFilename(instantTime, fileId)).toString();
+ return new StoragePath(
+ new StoragePath(partitionPath), getLogFilename(instantTime, fileId)).toString();
})
.collect(Collectors.toList());
List<String> expectedList = expectedResult.computeIfAbsent(
instantTime, k -> new ArrayList<>());
expectedList.addAll(fileNameList);
inputPathList.addAll(fileNameList.stream()
- .map(path -> new Path(basePath, path)).collect(Collectors.toList()));
+ .map(path -> new StoragePath(basePath, path)).collect(Collectors.toList()));
}
assertEquals(expectedResult,
@@ -155,11 +156,11 @@ public class TestRepairUtils {
List<String> fileListFromFs = partitionToFileIdAndNameMap.entrySet().stream()
.flatMap(entry ->
entry.getValue().stream()
- .map(fileInfo -> new Path(entry.getKey(), fileInfo.getValue()).toString())
+ .map(fileInfo -> new StoragePath(entry.getKey(), fileInfo.getValue()).toString())
.collect(Collectors.toList())
.stream()
).collect(Collectors.toList());
- String danglingFilePath = new Path("2022/01/02",
+ String danglingFilePath = new StoragePath("2022/01/02",
getBaseFilename(existingInstant.getTimestamp(), UUID.randomUUID().toString())).toString();
fileListFromFs.add(danglingFilePath);
// Existing instant
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
index 190d565c0a0..d726fab72dc 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
@@ -30,11 +30,11 @@ import org.apache.hudi.exception.HoodieValidationException;
import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
+import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
@@ -145,7 +145,7 @@ public class GenericRecordValidationTestUtils {
for (String path : paths) {
try (HoodieAvroHFileReaderImplBase reader = (HoodieAvroHFileReaderImplBase)
HoodieFileReaderFactory.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
- .getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, conf, new Path(path), HoodieFileFormat.HFILE)) {
+ .getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, conf, new StoragePath(path), HoodieFileFormat.HFILE)) {
valuesAsList.addAll(HoodieAvroHFileReaderImplBase.readAllRecords(reader)
.stream().map(e -> (GenericRecord) e).collect(Collectors.toList()));
} catch (IOException e) {
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java
index bf43202b152..a135a2b22e4 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java
@@ -44,13 +44,13 @@ import org.apache.hudi.io.storage.HoodieAvroParquetWriter;
import org.apache.hudi.io.storage.HoodieOrcConfig;
import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.orc.CompressionKind;
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.hadoop.ParquetWriter;
@@ -76,19 +76,23 @@ public class HoodieWriteableTestTable extends HoodieMetadataTestTable {
protected final BloomFilter filter;
protected final boolean populateMetaFields;
- protected HoodieWriteableTestTable(String basePath, FileSystem fs, HoodieTableMetaClient metaClient,
+ protected HoodieWriteableTestTable(String basePath, HoodieStorage storage,
+ HoodieTableMetaClient metaClient,
Schema schema, BloomFilter filter) {
- this(basePath, fs, metaClient, schema, filter, null);
+ this(basePath, storage, metaClient, schema, filter, null);
}
- protected HoodieWriteableTestTable(String basePath, FileSystem fs, HoodieTableMetaClient metaClient, Schema schema,
+ protected HoodieWriteableTestTable(String basePath, HoodieStorage storage,
+ HoodieTableMetaClient metaClient, Schema schema,
BloomFilter filter, HoodieTableMetadataWriter metadataWriter) {
- this(basePath, fs, metaClient, schema, filter, metadataWriter, Option.empty());
+ this(basePath, storage, metaClient, schema, filter, metadataWriter, Option.empty());
}
- protected HoodieWriteableTestTable(String basePath, FileSystem fs, HoodieTableMetaClient metaClient, Schema schema,
- BloomFilter filter, HoodieTableMetadataWriter metadataWriter, Option<HoodieEngineContext> context) {
- super(basePath, fs, metaClient, metadataWriter, context);
+ protected HoodieWriteableTestTable(String basePath, HoodieStorage storage,
+ HoodieTableMetaClient metaClient, Schema schema,
+ BloomFilter filter, HoodieTableMetadataWriter metadataWriter,
+ Option<HoodieEngineContext> context) {
+ super(basePath, storage, metaClient, metadataWriter, context);
this.schema = schema;
this.filter = filter;
this.populateMetaFields = metaClient.getTableConfig().populateMetaFields();
@@ -104,14 +108,15 @@ public class HoodieWriteableTestTable extends HoodieMetadataTestTable {
return (HoodieWriteableTestTable) super.forCommit(instantTime);
}
- public Path withInserts(String partition, String fileId, List<HoodieRecord> records, TaskContextSupplier contextSupplier) throws Exception {
+ public StoragePath withInserts(String partition, String fileId, List<HoodieRecord> records,
+ TaskContextSupplier contextSupplier) throws Exception {
FileCreateUtils.createPartitionMetaFile(basePath, partition);
String fileName = baseFileName(currentInstantTime, fileId);
- Path baseFilePath = new Path(Paths.get(basePath, partition, fileName).toString());
- if (this.fs.exists(baseFilePath)) {
+ StoragePath baseFilePath = new StoragePath(Paths.get(basePath, partition, fileName).toString());
+ if (storage.exists(baseFilePath)) {
LOG.warn("Deleting the existing base file " + baseFilePath);
- this.fs.delete(baseFilePath, true);
+ storage.deleteFile(baseFilePath);
}
if (HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().equals(HoodieFileFormat.PARQUET)) {
@@ -121,7 +126,7 @@ public class HoodieWriteableTestTable extends HoodieMetadataTestTable {
ParquetWriter.DEFAULT_BLOCK_SIZE, ParquetWriter.DEFAULT_PAGE_SIZE, 120 * 1024 * 1024,
new Configuration(), Double.parseDouble(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION.defaultValue()), true);
try (HoodieAvroParquetWriter writer = new HoodieAvroParquetWriter(
- new Path(Paths.get(basePath, partition, fileName).toString()), config, currentInstantTime,
+ new StoragePath(Paths.get(basePath, partition, fileName).toString()), config, currentInstantTime,
contextSupplier, populateMetaFields)) {
int seqId = 1;
for (HoodieRecord record : records) {
@@ -144,7 +149,7 @@ public class HoodieWriteableTestTable extends HoodieMetadataTestTable {
HoodieOrcConfig config = new HoodieOrcConfig(conf, CompressionKind.ZLIB, orcStripSize, orcBlockSize, maxFileSize, filter);
try (HoodieAvroOrcWriter writer = new HoodieAvroOrcWriter(
currentInstantTime,
- new Path(Paths.get(basePath, partition, fileName).toString()),
+ new StoragePath(Paths.get(basePath, partition, fileName).toString()),
config, schema, contextSupplier)) {
int seqId = 1;
for (HoodieRecord record : records) {
@@ -168,15 +173,17 @@ public class HoodieWriteableTestTable extends HoodieMetadataTestTable {
}
private Pair<String, HoodieLogFile> appendRecordsToLogFile(String partitionPath, String fileId, List<HoodieRecord> records) throws Exception {
- try (HoodieLogFormat.Writer logWriter = HoodieLogFormat.newWriterBuilder().onParentPath(new Path(basePath, partitionPath))
+ try (HoodieLogFormat.Writer logWriter = HoodieLogFormat.newWriterBuilder()
+ .onParentPath(new StoragePath(basePath, partitionPath))
.withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId(fileId)
- .withDeltaCommit(currentInstantTime).withFs(fs).build()) {
+ .withDeltaCommit(currentInstantTime).withStorage(storage).build()) {
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, currentInstantTime);
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
logWriter.appendBlock(new HoodieAvroDataBlock(records.stream().map(r -> {
try {
- GenericRecord val = (GenericRecord) ((HoodieRecordPayload) r.getData()).getInsertValue(schema).get();
+ GenericRecord val =
+ (GenericRecord) ((HoodieRecordPayload) r.getData()).getInsertValue(schema).get();
HoodieAvroUtils.addHoodieKeyToRecord(val, r.getRecordKey(), r.getPartitionPath(), "");
return (IndexedRecord) val;
} catch (IOException e) {
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/DFSProvider.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/DFSProvider.java
index 62b48cbf78b..d7ff2d39f2f 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/DFSProvider.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/DFSProvider.java
@@ -19,15 +19,16 @@
package org.apache.hudi.testutils.providers;
+import org.apache.hudi.storage.HoodieStorage;
+
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
public interface DFSProvider {
MiniDFSCluster dfsCluster();
- DistributedFileSystem dfs();
+ HoodieStorage hoodieStorage();
Path dfsBasePath();
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/HoodieMetaClientProvider.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/HoodieMetaClientProvider.java
index 721cc5e7c5b..f000b86f1ba 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/HoodieMetaClientProvider.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/HoodieMetaClientProvider.java
@@ -25,11 +25,12 @@ import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.table.view.SyncableFileSystemView;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
import java.io.IOException;
+import java.util.List;
import java.util.Properties;
public interface HoodieMetaClientProvider {
@@ -37,8 +38,9 @@ public interface HoodieMetaClientProvider {
HoodieTableMetaClient getHoodieMetaClient(Configuration hadoopConf, String basePath, Properties props) throws IOException;
default HoodieTableFileSystemView getHoodieTableFileSystemView(
- HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline, FileStatus[] fileStatuses) {
- return new HoodieTableFileSystemView(metaClient, visibleActiveTimeline, fileStatuses);
+ HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline,
+ List<StoragePathInfo> pathInfoList) {
+ return new HoodieTableFileSystemView(metaClient, visibleActiveTimeline, pathInfoList);
}
default SyncableFileSystemView getFileSystemViewWithUnCommittedSlices(HoodieTableMetaClient metaClient) {
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java
index bf7a3e33bf0..2d4e87c52e6 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java
@@ -36,11 +36,10 @@ import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.testutils.MetadataMergeWriteStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
@@ -166,31 +165,37 @@ public class HoodieWriterClientTestHarness extends HoodieCommonTestHarness {
return builder;
}
- public void assertPartitionMetadataForRecords(String basePath, List<HoodieRecord> inputRecords, FileSystem fs) throws IOException {
+ public void assertPartitionMetadataForRecords(String basePath, List<HoodieRecord> inputRecords,
+ HoodieStorage storage) throws IOException {
Set<String> partitionPathSet = inputRecords.stream()
.map(HoodieRecord::getPartitionPath)
.collect(Collectors.toSet());
- assertPartitionMetadata(basePath, partitionPathSet.stream().toArray(String[]::new), fs);
+ assertPartitionMetadata(basePath, partitionPathSet.stream().toArray(String[]::new), storage);
}
- public void assertPartitionMetadataForKeys(String basePath, List<HoodieKey> inputKeys, FileSystem fs) throws IOException {
+ public void assertPartitionMetadataForKeys(String basePath, List<HoodieKey> inputKeys,
+ HoodieStorage storage) throws IOException {
Set<String> partitionPathSet = inputKeys.stream()
.map(HoodieKey::getPartitionPath)
.collect(Collectors.toSet());
- assertPartitionMetadata(basePath, partitionPathSet.stream().toArray(String[]::new), fs);
+ assertPartitionMetadata(basePath, partitionPathSet.stream().toArray(String[]::new), storage);
}
/**
* Ensure presence of partition meta-data at known depth.
*
* @param partitionPaths Partition paths to check
- * @param fs File System
+ * @param storage {@link HoodieStorage} instance.
* @throws IOException in case of error
*/
- public static void assertPartitionMetadata(String basePath, String[] partitionPaths, FileSystem fs) throws IOException {
+ public static void assertPartitionMetadata(String basePath, String[] partitionPaths,
+ HoodieStorage storage) throws IOException {
for (String partitionPath : partitionPaths) {
- assertTrue(HoodiePartitionMetadata.hasPartitionMetadata(fs, new Path(basePath, partitionPath)));
- HoodiePartitionMetadata pmeta = new HoodiePartitionMetadata(fs, new Path(basePath, partitionPath));
+ assertTrue(
+ HoodiePartitionMetadata.hasPartitionMetadata(
+ storage, new StoragePath(basePath, partitionPath)));
+ HoodiePartitionMetadata pmeta =
+ new HoodiePartitionMetadata(storage, new StoragePath(basePath, partitionPath));
pmeta.readFromFS();
assertEquals(HoodieTestDataGenerator.DEFAULT_PARTITION_DEPTH, pmeta.getPartitionDepth());
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkAppendHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkAppendHandle.java
index 3dc76ed435e..9f99a1624d3 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkAppendHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkAppendHandle.java
@@ -22,11 +22,11 @@ import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -120,7 +120,7 @@ public class FlinkAppendHandle<T, I, K, O>
}
@Override
- public Path getWritePath() {
+ public StoragePath getWritePath() {
return writer.getLogFile().getPath();
}
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkConcatAndReplaceHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkConcatAndReplaceHandle.java
index 6ba7ac4d005..d69244fa1b4 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkConcatAndReplaceHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkConcatAndReplaceHandle.java
@@ -18,13 +18,14 @@
package org.apache.hudi.io;
-import org.apache.avro.Schema;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieUpsertException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
+
+import org.apache.avro.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,7 +48,7 @@ public class FlinkConcatAndReplaceHandle<T, I, K, O>
public FlinkConcatAndReplaceHandle(HoodieWriteConfig config, String instantTime, HoodieTable<T, I, K, O> hoodieTable,
Iterator<HoodieRecord<T>> recordItr, String partitionPath, String fileId,
- TaskContextSupplier taskContextSupplier, Path basePath) {
+ TaskContextSupplier taskContextSupplier, StoragePath basePath) {
super(config, instantTime, hoodieTable, Collections.emptyIterator(), partitionPath, fileId, taskContextSupplier, basePath);
this.recordItr = recordItr;
}
@@ -62,7 +63,8 @@ public class FlinkConcatAndReplaceHandle<T, I, K, O>
try {
fileWriter.write(key, oldRecord, writeSchema);
} catch (IOException | RuntimeException e) {
- String errMsg = String.format("Failed to write old record into new file for key %s from old file %s to new file %s with writerSchema %s",
+ String errMsg = String.format(
+ "Failed to write old record into new file for key %s from old file %s to new file %s with writerSchema %s",
key, getOldFilePath(), newFilePath, writeSchemaWithMetaFields.toString(true));
LOG.debug("Old record is " + oldRecord);
throw new HoodieUpsertException(errMsg, e);
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkConcatHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkConcatHandle.java
index 4f5f522df40..df3c178f549 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkConcatHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkConcatHandle.java
@@ -18,12 +18,13 @@
package org.apache.hudi.io;
-import org.apache.avro.Schema;
import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.table.HoodieTable;
+
+import org.apache.avro.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,7 +62,8 @@ public class FlinkConcatHandle<T, I, K, O>
try {
fileWriter.write(key, oldRecord, oldSchema);
} catch (IOException | RuntimeException e) {
- String errMsg = String.format("Failed to write old record into new file for key %s from old file %s to new file %s with writerSchema %s",
+ String errMsg = String.format(
+ "Failed to write old record into new file for key %s from old file %s to new file %s with writerSchema %s",
key, getOldFilePath(), newFilePath, oldSchema.toString(true));
LOG.debug("Old record is " + oldRecord);
throw new HoodieUpsertException(errMsg, e);
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkCreateHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkCreateHandle.java
index 32f251cc565..1cd117d2f0b 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkCreateHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkCreateHandle.java
@@ -25,12 +25,12 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.avro.Schema;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -89,11 +89,11 @@ public class FlinkCreateHandle<T, I, K, O>
final String lastWriteToken = FSUtils.makeWriteToken(getPartitionId(), getStageId(), lastAttemptId);
final String lastDataFileName = FSUtils.makeBaseFileName(instantTime,
lastWriteToken, this.fileId, hoodieTable.getBaseFileExtension());
- final Path path = makeNewFilePath(partitionPath, lastDataFileName);
+ final StoragePath path = makeNewFilePath(partitionPath, lastDataFileName);
try {
- if (fs.exists(path)) {
+ if (storage.exists(path)) {
LOG.info("Deleting invalid INSERT file due to task retry: " + lastDataFileName);
- fs.delete(path, false);
+ storage.deleteFile(path);
}
} catch (IOException e) {
throw new HoodieException("Error while deleting the INSERT file due to task retry: " + lastDataFileName, e);
@@ -107,16 +107,16 @@ public class FlinkCreateHandle<T, I, K, O>
}
@Override
- public Path makeNewPath(String partitionPath) {
- Path path = super.makeNewPath(partitionPath);
+ public StoragePath makeNewPath(String partitionPath) {
+ StoragePath path = super.makeNewPath(partitionPath);
// If the data file already exists, it means the write task write new data bucket multiple times
// in one hoodie commit, rolls over to a new name instead.
// Write to a new file which behaves like a different task write.
try {
int rollNumber = 0;
- while (fs.exists(path)) {
- Path existing = path;
+ while (storage.exists(path)) {
+ StoragePath existing = path;
path = newFilePathWithRollover(rollNumber++);
LOG.warn("Duplicate write for INSERT bucket with path: " + existing + ", rolls over to new path: " + path);
}
@@ -134,7 +134,7 @@ public class FlinkCreateHandle<T, I, K, O>
/**
* Use the writeToken + "-" + rollNumber as the new writeToken of a mini-batch write.
*/
- private Path newFilePathWithRollover(int rollNumber) {
+ private StoragePath newFilePathWithRollover(int rollNumber) {
final String dataFileName = FSUtils.makeBaseFileName(instantTime, writeToken + "-" + rollNumber, fileId,
hoodieTable.getBaseFileExtension());
return makeNewFilePath(partitionPath, dataFileName);
@@ -159,7 +159,7 @@ public class FlinkCreateHandle<T, I, K, O>
} catch (Throwable throwable) {
LOG.warn("Error while trying to dispose the CREATE handle", throwable);
try {
- fs.delete(path, false);
+ storage.deleteFile(path);
LOG.info("Deleting the intermediate CREATE data file: " + path + " success!");
} catch (IOException e) {
// logging a warning and ignore the exception.
@@ -169,7 +169,7 @@ public class FlinkCreateHandle<T, I, K, O>
}
@Override
- public Path getWritePath() {
+ public StoragePath getWritePath() {
return path;
}
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeAndReplaceHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeAndReplaceHandle.java
index 28d810ba350..fa91350274c 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeAndReplaceHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeAndReplaceHandle.java
@@ -27,11 +27,11 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,7 +64,7 @@ public class FlinkMergeAndReplaceHandle<T, I, K, O>
public FlinkMergeAndReplaceHandle(HoodieWriteConfig config, String instantTime, HoodieTable<T, I, K, O> hoodieTable,
Iterator<HoodieRecord<T>> recordItr, String partitionPath, String fileId,
- TaskContextSupplier taskContextSupplier, Path basePath) {
+ TaskContextSupplier taskContextSupplier, StoragePath basePath) {
super(config, instantTime, hoodieTable, recordItr, partitionPath, fileId, taskContextSupplier,
new HoodieBaseFile(basePath.toString()), Option.empty());
// delete invalid data files generated by task retry.
@@ -91,11 +91,11 @@ public class FlinkMergeAndReplaceHandle<T, I, K, O>
final String lastWriteToken = FSUtils.makeWriteToken(getPartitionId(), getStageId(), lastAttemptId);
final String lastDataFileName = FSUtils.makeBaseFileName(instantTime,
lastWriteToken, this.fileId, hoodieTable.getBaseFileExtension());
- final Path path = makeNewFilePath(partitionPath, lastDataFileName);
+ final StoragePath path = makeNewFilePath(partitionPath, lastDataFileName);
try {
- if (fs.exists(path)) {
+ if (storage.exists(path)) {
LOG.info("Deleting invalid MERGE and REPLACE base file due to task retry: " + lastDataFileName);
- fs.delete(path, false);
+ storage.deleteFile(path);
}
} catch (IOException e) {
throw new HoodieException("Error while deleting the MERGE and REPLACE base file due to task retry: " + lastDataFileName, e);
@@ -121,11 +121,12 @@ public class FlinkMergeAndReplaceHandle<T, I, K, O>
super.makeOldAndNewFilePaths(partitionPath, oldFileName, newFileName);
try {
int rollNumber = 0;
- while (fs.exists(newFilePath)) {
- Path oldPath = newFilePath;
+ while (storage.exists(newFilePath)) {
+ StoragePath oldPath = newFilePath;
newFileName = newFileNameWithRollover(rollNumber++);
newFilePath = makeNewFilePath(partitionPath, newFileName);
- LOG.warn("Duplicate write for MERGE and REPLACE handle with path: " + oldPath + ", rolls over to new path: " + newFilePath);
+ LOG.warn("Duplicate write for MERGE and REPLACE handle with path: " + oldPath
+ + ", rolls over to new path: " + newFilePath);
}
} catch (IOException e) {
throw new HoodieException("Checking existing path for merge and replace handle error: " + newFilePath, e);
@@ -143,7 +144,7 @@ public class FlinkMergeAndReplaceHandle<T, I, K, O>
@Override
protected void setWriteStatusPath() {
// should still report the old file path.
- writeStatus.getStat().setPath(new Path(config.getBasePath()), oldFilePath);
+ writeStatus.getStat().setPath(new StoragePath(config.getBasePath()), oldFilePath);
}
boolean needsUpdateLocation() {
@@ -159,12 +160,12 @@ public class FlinkMergeAndReplaceHandle<T, I, K, O>
}
// The file visibility should be kept by the configured ConsistencyGuard instance.
try {
- fs.delete(oldFilePath, false);
+ storage.deleteFile(oldFilePath);
} catch (IOException e) {
throw new HoodieIOException("Error while cleaning the old base file: " + oldFilePath, e);
}
try {
- fs.rename(newFilePath, oldFilePath);
+ storage.rename(newFilePath, oldFilePath);
} catch (IOException e) {
throw new HoodieIOException("Error while renaming the temporary rollover file: "
+ newFilePath + " to old base file name: " + oldFilePath, e);
@@ -192,7 +193,7 @@ public class FlinkMergeAndReplaceHandle<T, I, K, O>
} catch (Throwable throwable) {
LOG.warn("Error while trying to dispose the MERGE handle", throwable);
try {
- fs.delete(newFilePath, false);
+ storage.deleteFile(newFilePath);
LOG.info("Deleting the intermediate MERGE and REPLACE data file: " + newFilePath + " success!");
} catch (IOException e) {
// logging a warning and ignore the exception.
@@ -202,7 +203,7 @@ public class FlinkMergeAndReplaceHandle<T, I, K, O>
}
@Override
- public Path getWritePath() {
+ public StoragePath getWritePath() {
return oldFilePath;
}
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeAndReplaceHandleWithChangeLog.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeAndReplaceHandleWithChangeLog.java
index 85fb5a43504..b1049e1d73c 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeAndReplaceHandleWithChangeLog.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeAndReplaceHandleWithChangeLog.java
@@ -27,12 +27,13 @@ import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.table.cdc.HoodieCDCUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,14 +56,14 @@ public class FlinkMergeAndReplaceHandleWithChangeLog<T, I, K, O>
public FlinkMergeAndReplaceHandleWithChangeLog(HoodieWriteConfig config, String instantTime, HoodieTable<T, I, K, O> hoodieTable,
Iterator<HoodieRecord<T>> recordItr, String partitionPath, String fileId,
- TaskContextSupplier taskContextSupplier, Path basePath) {
+ TaskContextSupplier taskContextSupplier, StoragePath basePath) {
super(config, instantTime, hoodieTable, recordItr, partitionPath, fileId, taskContextSupplier, basePath);
this.cdcLogger = new HoodieCDCLogger(
instantTime,
config,
hoodieTable.getMetaClient().getTableConfig(),
partitionPath,
- getFileSystem(),
+ (FileSystem) getStorage().getFileSystem(),
getWriterSchema(),
createLogWriter(instantTime, HoodieCDCUtils.CDC_LOGFILE_SUFFIX),
IOUtils.getMaxMemoryPerPartitionMerge(taskContextSupplier, config));
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandle.java
index 4cea72d16ab..c9c53ab108c 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandle.java
@@ -26,11 +26,11 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,7 +61,7 @@ public class FlinkMergeHandle<T, I, K, O>
/**
* Records the rolled over file paths.
*/
- private List<Path> rolloverPaths;
+ private List<StoragePath> rolloverPaths;
public FlinkMergeHandle(HoodieWriteConfig config, String instantTime, HoodieTable<T, I, K, O> hoodieTable,
Iterator<HoodieRecord<T>> recordItr, String partitionPath, String fileId,
@@ -95,7 +95,7 @@ public class FlinkMergeHandle<T, I, K, O>
final String lastWriteToken = FSUtils.makeWriteToken(getPartitionId(), getStageId(), lastAttemptId);
final String lastDataFileName = FSUtils.makeBaseFileName(instantTime,
lastWriteToken, this.fileId, hoodieTable.getBaseFileExtension());
- final Path path = makeNewFilePath(partitionPath, lastDataFileName);
+ final StoragePath path = makeNewFilePath(partitionPath, lastDataFileName);
if (path.equals(oldFilePath)) {
// In some rare cases, the old attempt file is used as the old base file to merge
// because the flink index eagerly records that.
@@ -105,9 +105,9 @@ public class FlinkMergeHandle<T, I, K, O>
return;
}
try {
- if (fs.exists(path)) {
+ if (storage.exists(path)) {
LOG.info("Deleting invalid MERGE base file due to task retry: " + lastDataFileName);
- fs.delete(path, false);
+ storage.deleteFile(path);
}
} catch (IOException e) {
throw new HoodieException("Error while deleting the MERGE base file due to task retry: " + lastDataFileName, e);
@@ -134,10 +134,10 @@ public class FlinkMergeHandle<T, I, K, O>
rolloverPaths = new ArrayList<>();
try {
int rollNumber = 0;
- while (fs.exists(newFilePath)) {
+ while (storage.exists(newFilePath)) {
// in case there is empty file because of task failover attempt.
- if (fs.getFileStatus(newFilePath).getLen() <= 0) {
- fs.delete(newFilePath, false);
+ if (storage.getPathInfo(newFilePath).getLength() <= 0) {
+ storage.deleteFile(newFilePath);
LOG.warn("Delete empty write file for MERGE bucket: " + newFilePath);
break;
}
@@ -145,7 +145,8 @@ public class FlinkMergeHandle<T, I, K, O>
rolloverPaths.add(newFilePath);
newFileName = newFileNameWithRollover(rollNumber++);
newFilePath = makeNewFilePath(partitionPath, newFileName);
- LOG.warn("Duplicate write for MERGE bucket with path: " + oldFilePath + ", rolls over to new path: " + newFilePath);
+ LOG.warn("Duplicate write for MERGE bucket with path: " + oldFilePath
+ + ", rolls over to new path: " + newFilePath);
}
} catch (IOException e) {
throw new HoodieException("Checking existing path for merge handle error: " + newFilePath, e);
@@ -163,7 +164,7 @@ public class FlinkMergeHandle<T, I, K, O>
@Override
protected void setWriteStatusPath() {
// if there was rollover, should set up the path as the initial new file path.
- writeStatus.getStat().setPath(new Path(config.getBasePath()), getWritePath());
+ writeStatus.getStat().setPath(new StoragePath(config.getBasePath()), getWritePath());
}
@Override
@@ -190,19 +191,20 @@ public class FlinkMergeHandle<T, I, K, O>
return;
}
- for (Path path : rolloverPaths) {
+ for (StoragePath path : rolloverPaths) {
try {
- fs.delete(path, false);
+ storage.deleteFile(path);
LOG.info("Delete the rollover data file: " + path + " success!");
} catch (IOException e) {
throw new HoodieIOException("Error when clean the temporary rollover data file: " + path, e);
}
}
- final Path desiredPath = rolloverPaths.get(0);
+ final StoragePath desiredPath = rolloverPaths.get(0);
try {
- fs.rename(newFilePath, desiredPath);
+ storage.rename(newFilePath, desiredPath);
} catch (IOException e) {
- throw new HoodieIOException("Error when rename the temporary roll file: " + newFilePath + " to: " + desiredPath, e);
+ throw new HoodieIOException(
+ "Error when rename the temporary roll file: " + newFilePath + " to: " + desiredPath, e);
}
}
@@ -216,7 +218,7 @@ public class FlinkMergeHandle<T, I, K, O>
} catch (Throwable throwable) {
LOG.warn("Error while trying to dispose the MERGE handle", throwable);
try {
- fs.delete(newFilePath, false);
+ storage.deleteFile(newFilePath);
LOG.info("Deleting the intermediate MERGE data file: " + newFilePath + " success!");
} catch (IOException e) {
// logging a warning and ignore the exception.
@@ -226,7 +228,7 @@ public class FlinkMergeHandle<T, I, K, O>
}
@Override
- public Path getWritePath() {
+ public StoragePath getWritePath() {
return rolloverPaths.size() > 0 ? rolloverPaths.get(0) : newFilePath;
}
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandleWithChangeLog.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandleWithChangeLog.java
index 92335d0965d..040c7d5b514 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandleWithChangeLog.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandleWithChangeLog.java
@@ -32,6 +32,7 @@ import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
+import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,7 +61,7 @@ public class FlinkMergeHandleWithChangeLog<T, I, K, O>
config,
hoodieTable.getMetaClient().getTableConfig(),
partitionPath,
- getFileSystem(),
+ (FileSystem) getStorage().getFileSystem(),
getWriterSchema(),
createLogWriter(instantTime, HoodieCDCUtils.CDC_LOGFILE_SUFFIX),
IOUtils.getMaxMemoryPerPartitionMerge(taskContextSupplier, config));
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkWriteHandleFactory.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkWriteHandleFactory.java
index 1842e827fab..188a92663ee 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkWriteHandleFactory.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkWriteHandleFactory.java
@@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieRecordLocation;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hadoop.fs.Path;
@@ -107,8 +108,8 @@ public class FlinkWriteHandleFactory {
Path writePath = bucketToHandles.get(fileID);
if (writePath != null) {
HoodieWriteHandle<?, ?, ?, ?> writeHandle =
- createReplaceHandle(config, instantTime, table, recordItr, partitionPath, fileID, writePath);
- bucketToHandles.put(fileID, ((MiniBatchHandle) writeHandle).getWritePath()); // override with new replace handle
+ createReplaceHandle(config, instantTime, table, recordItr, partitionPath, fileID, new StoragePath(writePath.toUri()));
+ bucketToHandles.put(fileID, new Path(((MiniBatchHandle) writeHandle).getWritePath().toUri())); // override with new replace handle
return writeHandle;
}
@@ -119,7 +120,7 @@ public class FlinkWriteHandleFactory {
} else {
writeHandle = createMergeHandle(config, instantTime, table, recordItr, partitionPath, fileID);
}
- bucketToHandles.put(fileID, ((MiniBatchHandle) writeHandle).getWritePath());
+ bucketToHandles.put(fileID, new Path(((MiniBatchHandle) writeHandle).getWritePath().toUri()));
return writeHandle;
}
@@ -130,7 +131,7 @@ public class FlinkWriteHandleFactory {
Iterator<HoodieRecord<T>> recordItr,
String partitionPath,
String fileId,
- Path basePath);
+ StoragePath basePath);
protected abstract HoodieWriteHandle<?, ?, ?, ?> createMergeHandle(
HoodieWriteConfig config,
@@ -161,7 +162,7 @@ public class FlinkWriteHandleFactory {
Iterator<HoodieRecord<T>> recordItr,
String partitionPath,
String fileId,
- Path basePath) {
+ StoragePath basePath) {
return new FlinkMergeAndReplaceHandle<>(config, instantTime, table, recordItr, partitionPath, fileId,
table.getTaskContextSupplier(), basePath);
}
@@ -199,7 +200,7 @@ public class FlinkWriteHandleFactory {
Iterator<HoodieRecord<T>> recordItr,
String partitionPath,
String fileId,
- Path basePath) {
+ StoragePath basePath) {
return new FlinkConcatAndReplaceHandle<>(config, instantTime, table, recordItr, partitionPath, fileId,
table.getTaskContextSupplier(), basePath);
}
@@ -237,7 +238,7 @@ public class FlinkWriteHandleFactory {
Iterator<HoodieRecord<T>> recordItr,
String partitionPath,
String fileId,
- Path basePath) {
+ StoragePath basePath) {
return new FlinkMergeAndReplaceHandleWithChangeLog<>(config, instantTime, table, recordItr, partitionPath, fileId,
table.getTaskContextSupplier(), basePath);
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/MiniBatchHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/MiniBatchHandle.java
index 91b8f6630c7..c70966fb354 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/MiniBatchHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/MiniBatchHandle.java
@@ -18,7 +18,7 @@
package org.apache.hudi.io;
-import org.apache.hadoop.fs.Path;
+import org.apache.hudi.storage.StoragePath;
/**
* Hoodie write handle that supports write as mini-batch.
@@ -42,5 +42,5 @@ public interface MiniBatchHandle {
/**
* Returns the write file path.
*/
- Path getWritePath();
+ StoragePath getWritePath();
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
index 475d0efc582..0d24576cfbd 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
@@ -33,13 +33,14 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieInsertException;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.logical.RowType;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,7 +69,7 @@ public class HoodieRowDataCreateHandle implements Serializable {
private final Path path;
private final String fileId;
private final boolean preserveHoodieMetadata;
- private final FileSystem fs;
+ private final HoodieStorage storage;
protected final WriteStatus writeStatus;
private final HoodieRecordLocation newRecordLocation;
@@ -88,7 +89,7 @@ public class HoodieRowDataCreateHandle implements Serializable {
this.newRecordLocation = new HoodieRecordLocation(instantTime, fileId);
this.preserveHoodieMetadata = preserveHoodieMetadata;
this.currTimer = HoodieTimer.start();
- this.fs = table.getMetaClient().getFs();
+ this.storage = table.getMetaClient().getStorage();
this.path = makeNewPath(partitionPath);
this.writeStatus = new WriteStatus(table.shouldTrackSuccessRecords(),
@@ -99,9 +100,9 @@ public class HoodieRowDataCreateHandle implements Serializable {
try {
HoodiePartitionMetadata partitionMetadata =
new HoodiePartitionMetadata(
- fs,
+ storage,
instantTime,
- new Path(writeConfig.getBasePath()),
+ new StoragePath(writeConfig.getBasePath()),
FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath),
table.getPartitionMetafileFormat());
partitionMetadata.trySave(taskPartitionId);
@@ -170,8 +171,9 @@ public class HoodieRowDataCreateHandle implements Serializable {
stat.setNumInserts(writeStatus.getTotalRecords());
stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT);
stat.setFileId(fileId);
- stat.setPath(new Path(writeConfig.getBasePath()), path);
- long fileSizeInBytes = FSUtils.getFileSize(table.getMetaClient().getFs(), path);
+ stat.setPath(new StoragePath(writeConfig.getBasePath()), new StoragePath(path.toUri()));
+ long fileSizeInBytes = FSUtils.getFileSize(
+ table.getMetaClient().getStorage(), new StoragePath(path.toUri()));
stat.setTotalWriteBytes(fileSizeInBytes);
stat.setFileSizeInBytes(fileSizeInBytes);
stat.setTotalWriteErrors(writeStatus.getTotalErrorRecords());
@@ -186,10 +188,11 @@ public class HoodieRowDataCreateHandle implements Serializable {
}
private Path makeNewPath(String partitionPath) {
- Path path = FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath);
+ StoragePath path =
+ FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath);
try {
- if (!fs.exists(path)) {
- fs.mkdirs(path); // create a new partition as needed.
+ if (!storage.exists(path)) {
+ storage.createDirectory(path); // create a new partition as needed.
}
} catch (IOException e) {
throw new HoodieIOException("Failed to make dir " + path, e);
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java
index 816cec4f906..1bec707145c 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java
@@ -23,6 +23,7 @@ import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.io.storage.HoodieParquetConfig;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.flink.table.types.logical.RowType;
@@ -68,7 +69,7 @@ public class HoodieRowDataFileWriterFactory {
HoodieRowDataParquetWriteSupport writeSupport =
new HoodieRowDataParquetWriteSupport(table.getHadoopConf(), rowType, filter);
return new HoodieRowDataParquetWriter(
- path, new HoodieParquetConfig<>(
+ new StoragePath(path.toUri()), new HoodieParquetConfig<>(
writeSupport,
writeConfig.getParquetCompressionCodec(),
writeConfig.getParquetBlockSize(),
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java
index 099b0224791..8acd1ef9dd1 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java
@@ -18,10 +18,11 @@
package org.apache.hudi.io.storage.row;
-import org.apache.flink.table.data.RowData;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.io.storage.HoodieBaseParquetWriter;
import org.apache.hudi.io.storage.HoodieParquetConfig;
+import org.apache.hudi.storage.StoragePath;
+
+import org.apache.flink.table.data.RowData;
import java.io.IOException;
@@ -33,7 +34,7 @@ public class HoodieRowDataParquetWriter extends HoodieBaseParquetWriter<RowData>
private final HoodieRowDataParquetWriteSupport writeSupport;
- public HoodieRowDataParquetWriter(Path file, HoodieParquetConfig<HoodieRowDataParquetWriteSupport> parquetConfig)
+ public HoodieRowDataParquetWriter(StoragePath file, HoodieParquetConfig<HoodieRowDataParquetWriteSupport> parquetConfig)
throws IOException {
super(file, parquetConfig);
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkCopyOnWriteTable.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkCopyOnWriteTable.java
index 05779339780..1ea69d3a109 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkCopyOnWriteTable.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkCopyOnWriteTable.java
@@ -65,6 +65,7 @@ import org.apache.hudi.table.action.commit.FlinkUpsertCommitActionExecutor;
import org.apache.hudi.table.action.commit.FlinkUpsertPreppedCommitActionExecutor;
import org.apache.hudi.table.action.rollback.BaseRollbackPlanActionExecutor;
import org.apache.hudi.table.action.rollback.CopyOnWriteRollbackActionExecutor;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java
index 5fc6d8a807a..54c079b5166 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeletePartitionCommitActionExecutor.java
@@ -18,7 +18,6 @@
package org.apache.hudi.table.action.commit;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.utils.DeletePartitionUtils;
@@ -32,6 +31,7 @@ import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieDeletePartitionException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.WorkloadProfile;
import org.apache.hudi.table.WorkloadStat;
@@ -70,25 +70,29 @@ public class FlinkDeletePartitionCommitActionExecutor<T extends HoodieRecordPayl
context.setJobStatus(this.getClass().getSimpleName(), "Gather all file ids from all deleting partitions.");
Map<String, List<String>> partitionToReplaceFileIds =
context.parallelize(partitions).distinct().collectAsList()
- .stream().collect(Collectors.toMap(partitionPath -> partitionPath, this::getAllExistingFileIds));
+ .stream().collect(
+ Collectors.toMap(partitionPath -> partitionPath, this::getAllExistingFileIds));
HoodieWriteMetadata<List<WriteStatus>> result = new HoodieWriteMetadata<>();
result.setPartitionToReplaceFileIds(partitionToReplaceFileIds);
result.setIndexUpdateDuration(Duration.ofMillis(timer.endTimer()));
result.setWriteStatuses(Collections.emptyList());
// created requested
- HoodieInstant dropPartitionsInstant = new HoodieInstant(REQUESTED, REPLACE_COMMIT_ACTION, instantTime);
- if (!table.getMetaClient().getFs().exists(new Path(table.getMetaClient().getMetaPath(),
- dropPartitionsInstant.getFileName()))) {
- HoodieRequestedReplaceMetadata requestedReplaceMetadata = HoodieRequestedReplaceMetadata.newBuilder()
- .setOperationType(WriteOperationType.DELETE_PARTITION.name())
- .setExtraMetadata(extraMetadata.orElse(Collections.emptyMap()))
- .build();
+ HoodieInstant dropPartitionsInstant =
+ new HoodieInstant(REQUESTED, REPLACE_COMMIT_ACTION, instantTime);
+ if (!table.getMetaClient().getStorage().exists(new StoragePath(
+ table.getMetaClient().getMetaPath(), dropPartitionsInstant.getFileName()))) {
+ HoodieRequestedReplaceMetadata requestedReplaceMetadata =
+ HoodieRequestedReplaceMetadata.newBuilder()
+ .setOperationType(WriteOperationType.DELETE_PARTITION.name())
+ .setExtraMetadata(extraMetadata.orElse(Collections.emptyMap()))
+ .build();
table.getMetaClient().getActiveTimeline().saveToPendingReplaceCommit(dropPartitionsInstant,
TimelineMetadataUtils.serializeRequestedReplaceMetadata(requestedReplaceMetadata));
}
- this.saveWorkloadProfileMetadataToInflight(new WorkloadProfile(Pair.of(new HashMap<>(), new WorkloadStat())),
+ this.saveWorkloadProfileMetadataToInflight(
+ new WorkloadProfile(Pair.of(new HashMap<>(), new WorkloadStat())),
instantTime);
this.commitOnAutoCommit(result);
return result;
diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java
index f9a6c0d5ea1..c1572cef410 100644
--- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java
+++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/index/bloom/TestFlinkHoodieBloomIndex.java
@@ -34,13 +34,13 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.index.HoodieIndexUtils;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieFlinkTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.testutils.HoodieFlinkClientTestHarness;
import org.apache.hudi.testutils.HoodieFlinkWriteableTestTable;
import org.apache.avro.Schema;
-import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -247,7 +247,7 @@ public class TestFlinkHoodieBloomIndex extends HoodieFlinkClientTestHarness {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).build();
List<Pair<String, Long>> results = HoodieIndexUtils.filterKeysFromFile(
- new Path(java.nio.file.Paths.get(basePath, partition, filename).toString()), uuids, hadoopConf);
+ new StoragePath(java.nio.file.Paths.get(basePath, partition, filename).toString()), uuids, hadoopConf);
assertEquals(results.size(), 2);
assertTrue(results.get(0).getLeft().equals("1eb5b87a-1feh-4edd-87b4-6ec96dc405a0")
|| results.get(1).getLeft().equals("1eb5b87a-1feh-4edd-87b4-6ec96dc405a0"));
diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java
index e92bd6eddfa..707a715d47a 100644
--- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java
+++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java
@@ -33,12 +33,13 @@ import org.apache.hudi.common.table.log.HoodieLogFormat;
import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock;
import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,16 +57,21 @@ import java.util.stream.Collectors;
public class HoodieFlinkWriteableTestTable extends HoodieWriteableTestTable {
private static final Logger LOG = LoggerFactory.getLogger(HoodieFlinkWriteableTestTable.class);
- private HoodieFlinkWriteableTestTable(String basePath, org.apache.hadoop.fs.FileSystem fs, HoodieTableMetaClient metaClient, Schema schema, BloomFilter filter) {
- super(basePath, fs, metaClient, schema, filter);
+ private HoodieFlinkWriteableTestTable(String basePath, HoodieStorage storage,
+ HoodieTableMetaClient metaClient, Schema schema,
+ BloomFilter filter) {
+ super(basePath, storage, metaClient, schema, filter);
}
- public static HoodieFlinkWriteableTestTable of(HoodieTableMetaClient metaClient, Schema schema, BloomFilter filter) {
- return new HoodieFlinkWriteableTestTable(metaClient.getBasePathV2().toString(), metaClient.getRawFs(), metaClient, schema, filter);
+ public static HoodieFlinkWriteableTestTable of(HoodieTableMetaClient metaClient, Schema schema,
+ BloomFilter filter) {
+ return new HoodieFlinkWriteableTestTable(metaClient.getBasePathV2().toString(),
+ metaClient.getRawHoodieStorage(), metaClient, schema, filter);
}
public static HoodieFlinkWriteableTestTable of(HoodieTableMetaClient metaClient, Schema schema) {
- BloomFilter filter = BloomFilterFactory.createBloomFilter(10000, 0.0000001, -1, BloomFilterTypeCode.SIMPLE.name());
+ BloomFilter filter = BloomFilterFactory.createBloomFilter(10000, 0.0000001, -1,
+ BloomFilterTypeCode.SIMPLE.name());
return of(metaClient, schema, filter);
}
@@ -130,15 +136,17 @@ public class HoodieFlinkWriteableTestTable extends HoodieWriteableTestTable {
private Pair<String, HoodieLogFile> appendRecordsToLogFile(List<HoodieRecord> groupedRecords) throws Exception {
String partitionPath = groupedRecords.get(0).getPartitionPath();
HoodieRecordLocation location = groupedRecords.get(0).getCurrentLocation();
- try (HoodieLogFormat.Writer logWriter = HoodieLogFormat.newWriterBuilder().onParentPath(new Path(basePath, partitionPath))
+ try (HoodieLogFormat.Writer logWriter = HoodieLogFormat.newWriterBuilder()
+ .onParentPath(new StoragePath(basePath, partitionPath))
.withFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId(location.getFileId())
- .withDeltaCommit(location.getInstantTime()).withFs(fs).build()) {
+ .withDeltaCommit(location.getInstantTime()).withStorage(storage).build()) {
Map<HeaderMetadataType, String> header = new java.util.HashMap<>();
header.put(HeaderMetadataType.INSTANT_TIME, location.getInstantTime());
header.put(HeaderMetadataType.SCHEMA, schema.toString());
logWriter.appendBlock(new HoodieAvroDataBlock(groupedRecords.stream().map(r -> {
try {
- GenericRecord val = (GenericRecord) ((HoodieRecordPayload) r.getData()).getInsertValue(schema).get();
+ GenericRecord val =
+ (GenericRecord) ((HoodieRecordPayload) r.getData()).getInsertValue(schema).get();
HoodieAvroUtils.addHoodieKeyToRecord(val, r.getRecordKey(), r.getPartitionPath(), "");
return (IndexedRecord) val;
} catch (IOException e) {
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
index 70e8de465df..b7d8c277b82 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
@@ -44,13 +44,13 @@ import org.apache.hudi.execution.bulkinsert.JavaCustomColumnsSortPartitioner;
import org.apache.hudi.io.IOUtils;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.table.action.cluster.strategy.ClusteringExecutionStrategy;
import org.apache.avro.Schema;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -175,7 +175,7 @@ public abstract class JavaExecutionStrategy<T>
try {
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(config.getSchema()));
scanner = HoodieMergedLogRecordScanner.newBuilder()
- .withFileSystem(table.getMetaClient().getFs())
+ .withStorage(table.getMetaClient().getStorage())
.withBasePath(table.getMetaClient().getBasePath())
.withLogFilePaths(clusteringOp.getDeltaFilePaths())
.withReaderSchema(readerSchema)
@@ -193,7 +193,7 @@ public abstract class JavaExecutionStrategy<T>
baseFileReader = StringUtils.isNullOrEmpty(clusteringOp.getDataFilePath())
? Option.empty()
: Option.of(HoodieFileReaderFactory.getReaderFactory(recordType)
- .getFileReader(config, table.getHadoopConf(), new Path(clusteringOp.getDataFilePath())));
+ .getFileReader(config, table.getHadoopConf(), new StoragePath(clusteringOp.getDataFilePath())));
HoodieTableConfig tableConfig = table.getMetaClient().getTableConfig();
Iterator<HoodieRecord<T>> fileSliceReader = new HoodieFileSliceReader(baseFileReader, scanner, readerSchema, tableConfig.getPreCombineField(), writeConfig.getRecordMerger(),
tableConfig.getProps(),
@@ -222,7 +222,7 @@ public abstract class JavaExecutionStrategy<T>
List<HoodieRecord<T>> records = new ArrayList<>();
clusteringOps.forEach(clusteringOp -> {
try (HoodieFileReader baseFileReader = HoodieFileReaderFactory.getReaderFactory(recordType)
- .getFileReader(getHoodieTable().getConfig(), getHoodieTable().getHadoopConf(), new Path(clusteringOp.getDataFilePath()))) {
+ .getFileReader(getHoodieTable().getConfig(), getHoodieTable().getHadoopConf(), new StoragePath(clusteringOp.getDataFilePath()))) {
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(getWriteConfig().getSchema()));
Iterator<HoodieRecord> recordIterator = baseFileReader.getRecordIterator(readerSchema);
// NOTE: Record have to be cloned here to make sure if it holds low-level engine-specific
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaCopyOnWriteTable.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaCopyOnWriteTable.java
index 6e111f67da2..16e194e2b66 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaCopyOnWriteTable.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaCopyOnWriteTable.java
@@ -71,6 +71,7 @@ import org.apache.hudi.table.action.rollback.BaseRollbackPlanActionExecutor;
import org.apache.hudi.table.action.rollback.CopyOnWriteRollbackActionExecutor;
import org.apache.hudi.table.action.rollback.RestorePlanActionExecutor;
import org.apache.hudi.table.action.savepoint.SavepointActionExecutor;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java
index ac365e9f7c7..2523c244b37 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java
@@ -35,10 +35,9 @@ import org.apache.hudi.index.JavaHoodieIndexFactory;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.metadata.JavaHoodieBackedTableMetadataWriter;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.action.HoodieWriteMetadata;
-import org.apache.hadoop.fs.Path;
-
import java.io.IOException;
import java.util.List;
@@ -95,7 +94,7 @@ public abstract class HoodieJavaTable<T>
// delete metadata partitions corresponding to such indexes
deleteMetadataIndexIfNecessary();
try {
- if (isMetadataTableExists || metaClient.getFs().exists(new Path(
+ if (isMetadataTableExists || metaClient.getStorage().exists(new StoragePath(
HoodieTableMetadata.getMetadataTableBasePath(metaClient.getBasePath())))) {
isMetadataTableExists = true;
return Option.of(metadataWriter);
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java
index 84e4316d164..61cffe26d68 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java
@@ -41,12 +41,12 @@ import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.io.HoodieMergeHandleFactory;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.factory.HoodieAvroKeyGeneratorFactory;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.WorkloadProfile;
import org.apache.hudi.table.WorkloadStat;
import org.apache.hudi.table.action.HoodieWriteMetadata;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -99,7 +99,8 @@ public abstract class BaseJavaCommitActionExecutor<T> extends
HoodieTableMetaClient metaClient = table.getMetaClient();
HoodieInstant inflightInstant = new HoodieInstant(HoodieInstant.State.INFLIGHT, metaClient.getCommitActionType(), instantTime);
try {
- if (!metaClient.getFs().exists(new Path(metaClient.getMetaPath(), inflightInstant.getFileName()))) {
+ if (!metaClient.getStorage().exists(
+ new StoragePath(metaClient.getMetaPath(), inflightInstant.getFileName()))) {
throw new HoodieCommitException("Failed to commit " + instantTime + " unable to save inflight metadata ", e);
}
} catch (IOException ex) {
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
index f6d984f83f8..570a61875ab 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestHoodieJavaWriteClientInsert.java
@@ -37,6 +37,7 @@ import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hadoop.HoodieParquetInputFormat;
import org.apache.hudi.hadoop.utils.HoodieHiveUtils;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.testutils.HoodieJavaClientTestHarness;
import org.apache.avro.Schema;
@@ -171,7 +172,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(hadoopConf, filePath);
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(hadoopConf, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records1) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
@@ -203,7 +204,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
records1.addAll(records2);
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(hadoopConf, filePath);
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(hadoopConf, new StoragePath(filePath.toUri()));
int index = 0;
for (GenericRecord record : fileRecords) {
assertEquals(records1.get(index).getRecordKey(), record.get("_row_key").toString());
@@ -238,7 +239,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(hadoopConf, filePath);
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(hadoopConf, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records1) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
@@ -259,7 +260,7 @@ public class TestHoodieJavaWriteClientInsert extends HoodieJavaClientTestHarness
records1.addAll(records2);
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(hadoopConf, filePath);
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(hadoopConf, new StoragePath(filePath.toUri()));
assertEquals(fileRecords.size(), mergeAllowDuplicateOnInsertsEnable ? records1.size() : records2.size());
int index = 0;
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
index 736eee97e85..341084c8934 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
@@ -101,6 +101,7 @@ import org.apache.hudi.metadata.JavaHoodieBackedTableMetadataWriter;
import org.apache.hudi.metadata.MetadataPartitionType;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.table.HoodieJavaTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.HoodieWriteMetadata;
@@ -110,8 +111,6 @@ import org.apache.hudi.testutils.TestHoodieMetadataBase;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.schema.MessageType;
import org.junit.jupiter.api.AfterEach;
@@ -277,14 +276,21 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
assertFalse(partitions.contains(filteredDirectoryThree),
"Must not contain the filtered directory " + filteredDirectoryThree);
- FileStatus[] statuses = metadata(writeConfig, context).getAllFilesInPartition(new Path(basePath, "p1"));
- assertEquals(tableType == COPY_ON_WRITE ? 3 : 4, statuses.length);
- statuses = metadata(writeConfig, context).getAllFilesInPartition(new Path(basePath, "p2"));
- assertEquals(tableType == COPY_ON_WRITE ? 6 : 7, statuses.length);
- Map<String, FileStatus[]> partitionsToFilesMap = metadata(writeConfig, context).getAllFilesInPartitions(asList(basePath + "/p1", basePath + "/p2"));
+ List<StoragePathInfo> pathInfoList =
+ metadata(writeConfig, context).getAllFilesInPartition(new StoragePath(basePath,
+ "p1"));
+ assertEquals(tableType == COPY_ON_WRITE ? 3 : 4, pathInfoList.size());
+ pathInfoList =
+ metadata(writeConfig, context).getAllFilesInPartition(new StoragePath(basePath, "p2"));
+ assertEquals(tableType == COPY_ON_WRITE ? 6 : 7, pathInfoList.size());
+ Map<String, List<StoragePathInfo>> partitionsToFilesMap =
+ metadata(writeConfig, context).getAllFilesInPartitions(
+ asList(basePath + "/p1", basePath + "/p2"));
assertEquals(2, partitionsToFilesMap.size());
- assertEquals(tableType == COPY_ON_WRITE ? 3 : 4, partitionsToFilesMap.get(basePath + "/p1").length);
- assertEquals(tableType == COPY_ON_WRITE ? 6 : 7, partitionsToFilesMap.get(basePath + "/p2").length);
+ assertEquals(tableType == COPY_ON_WRITE ? 3 : 4,
+ partitionsToFilesMap.get(basePath + "/p1").size());
+ assertEquals(tableType == COPY_ON_WRITE ? 6 : 7,
+ partitionsToFilesMap.get(basePath + "/p2").size());
}
/**
@@ -538,7 +544,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
HoodieFileReaderFactory.getReaderFactory(HoodieRecordType.AVRO).getFileReader(
- writeConfig, context.getHadoopConf().get(), new Path(baseFile.getPath()));
+ writeConfig, context.getHadoopConf().get(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
records.forEach(entry -> {
if (populateMetaFields) {
@@ -625,10 +631,14 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
validateMetadata(testTable);
// Fetch compaction Commit file and rename to some other file. completed compaction meta file should have some serialized info that table interprets
// for future upserts. so, renaming the file here to some temp name and later renaming it back to same name.
- java.nio.file.Path metaFilePath = Paths.get(HoodieTestUtils.getCompleteInstantPath(metaClient.getFs(),
- new Path(metadataTableBasePath, METAFOLDER_NAME), metadataCompactionInstant.get(), HoodieTimeline.COMMIT_ACTION)
- .toUri());
- java.nio.file.Path tempFilePath = FileCreateUtils.renameFileToTemp(metaFilePath, metadataCompactionInstant.get());
+ java.nio.file.Path metaFilePath =
+ Paths.get(HoodieTestUtils.getCompleteInstantPath(metaClient.getStorage(),
+ new StoragePath(metadataTableBasePath, METAFOLDER_NAME),
+ metadataCompactionInstant.get(),
+ HoodieTimeline.COMMIT_ACTION)
+ .toUri());
+ java.nio.file.Path tempFilePath =
+ FileCreateUtils.renameFileToTemp(metaFilePath, metadataCompactionInstant.get());
metaClient.reloadActiveTimeline();
testTable = HoodieMetadataTestTable.of(metaClient, metadataWriter, Option.of(context));
@@ -659,8 +669,10 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// Fetch compaction Commit file and rename to some other file. completed compaction meta file should have some serialized info that table interprets
// for future upserts. so, renaming the file here to some temp name and later renaming it back to same name.
- metaFilePath = Paths.get(HoodieTestUtils.getCompleteInstantPath(metaClient.getFs(),
- new Path(metadataTableBasePath, METAFOLDER_NAME), metadataCompactionInstant.get(), HoodieTimeline.COMMIT_ACTION).toUri());
+ metaFilePath = Paths.get(HoodieTestUtils.getCompleteInstantPath(metaClient.getStorage(),
+ new StoragePath(metadataTableBasePath, METAFOLDER_NAME),
+ metadataCompactionInstant.get(),
+ HoodieTimeline.COMMIT_ACTION).toUri());
FileCreateUtils.renameFileToTemp(metaFilePath, metadataCompactionInstant.get());
validateMetadata(testTable);
@@ -713,9 +725,12 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
client.rollback(newCommitTime3);
// mimicing crash or making an inflight in metadata table.
- Path toDelete = HoodieTestUtils
- .getCompleteInstantPath(metaClient.getFs(), new Path(metaClient.getMetaPath() + "/metadata/.hoodie/"), newCommitTime2, HoodieTimeline.DELTA_COMMIT_ACTION);
- metaClient.getFs().delete(toDelete);
+ StoragePath toDelete = HoodieTestUtils.getCompleteInstantPath(
+ metaClient.getStorage(),
+ new StoragePath(metaClient.getMetaPath() + "/metadata" + "/" + ".hoodie/"),
+ newCommitTime2,
+ HoodieTimeline.DELTA_COMMIT_ACTION);
+ metaClient.getStorage().deleteDirectory(toDelete);
// re-ingest w/ same commit time.
records = dataGen.generateUniqueUpdates(newCommitTime3, 20);
@@ -724,16 +739,22 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
client.commit(newCommitTime3, writeStatuses);
// collect all commit meta files from metadata table.
- FileStatus[] metaFiles = metaClient.getFs().listStatus(new Path(metaClient.getMetaPath() + "/metadata/.hoodie"));
- List<FileStatus> commit3Files = Arrays.stream(metaFiles).filter(fileStatus ->
- fileStatus.getPath().getName().contains(newCommitTime3)
- && fileStatus.getPath().getName().contains(HoodieTimeline.DELTA_COMMIT_ACTION)).collect(Collectors.toList());
- List<FileStatus> rollbackFiles = Arrays.stream(metaFiles).filter(fileStatus ->
- fileStatus.getPath().getName().endsWith("." + HoodieTimeline.ROLLBACK_ACTION)).collect(Collectors.toList());
+ List<StoragePathInfo> metaFiles = metaClient.getStorage().listDirectEntries(
+ new StoragePath(metaClient.getMetaPath() + "/metadata/.hoodie"));
+ List<StoragePathInfo> commit3Files = metaFiles.stream()
+ .filter(fileInfo ->
+ fileInfo.getPath().getName().contains(newCommitTime3)
+ && fileInfo.getPath().getName().contains(HoodieTimeline.DELTA_COMMIT_ACTION))
+ .collect(Collectors.toList());
+ List<StoragePathInfo> rollbackFiles = metaFiles.stream()
+ .filter(fileStatus ->
+ fileStatus.getPath().getName().endsWith("." + HoodieTimeline.ROLLBACK_ACTION))
+ .collect(Collectors.toList());
// ensure commit2's delta commit in MDT has last mod time > the actual rollback for previous failed commit i.e. commit2.
// if rollback wasn't eager, rollback's last mod time will be lower than the commit3'd delta commit last mod time.
- assertTrue(commit3Files.get(0).getModificationTime() > rollbackFiles.get(0).getModificationTime());
+ assertTrue(
+ commit3Files.get(0).getModificationTime() > rollbackFiles.get(0).getModificationTime());
}
}
@@ -868,19 +889,23 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
*/
private void verifyMetadataRawRecords(HoodieTable table, List<HoodieLogFile> logFiles, boolean enableMetaFields) throws IOException {
for (HoodieLogFile logFile : logFiles) {
- FileStatus[] fsStatus = fs.listStatus(logFile.getPath());
- MessageType writerSchemaMsg = TableSchemaResolver.readSchemaFromLogFile(fs, logFile.getPath());
+ List<StoragePathInfo> pathInfoList = storage.listDirectEntries(logFile.getPath());
+ MessageType writerSchemaMsg = TableSchemaResolver.readSchemaFromLogFile(storage,
+ logFile.getPath());
if (writerSchemaMsg == null) {
// not a data block
continue;
}
Schema writerSchema = new AvroSchemaConverter().convert(writerSchemaMsg);
- try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(fs, new HoodieLogFile(fsStatus[0].getPath()), writerSchema)) {
+ try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(storage,
+ new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) {
while (logFileReader.hasNext()) {
HoodieLogBlock logBlock = logFileReader.next();
if (logBlock instanceof HoodieDataBlock) {
- try (ClosableIterator<HoodieRecord<IndexedRecord>> recordItr = ((HoodieDataBlock) logBlock).getRecordIterator(HoodieRecordType.AVRO)) {
+ try (
+ ClosableIterator<HoodieRecord<IndexedRecord>> recordItr = ((HoodieDataBlock) logBlock).getRecordIterator(
+ HoodieRecordType.AVRO)) {
recordItr.forEachRemaining(indexRecord -> {
final GenericRecord record = (GenericRecord) indexRecord.getData();
if (enableMetaFields) {
@@ -923,7 +948,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
schema = HoodieAvroUtils.addMetadataFields(schema);
}
HoodieMetadataLogRecordReader logRecordReader = HoodieMetadataLogRecordReader.newBuilder()
- .withFileSystem(metadataMetaClient.getFs())
+ .withStorage(metadataMetaClient.getStorage())
.withBasePath(metadataMetaClient.getBasePath())
.withLogFilePaths(logFilePaths)
.withLatestInstantTime(latestCommitTimestamp)
@@ -960,7 +985,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
HoodieFileReaderFactory.getReaderFactory(HoodieRecordType.AVRO).getFileReader(
- table.getConfig(), context.getHadoopConf().get(), new Path(baseFile.getPath()));
+ table.getConfig(), context.getHadoopConf().get(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
records.forEach(entry -> {
if (enableMetaFields) {
@@ -1208,23 +1233,27 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
validateMetadata(client);
// Metadata table should exist
- final Path metadataTablePath = new Path(getMetadataTableBasePath(writeConfig.getBasePath()));
- assertTrue(fs.exists(metadataTablePath));
+ final StoragePath metadataTablePath =
+ new StoragePath(getMetadataTableBasePath(writeConfig.getBasePath()));
+ assertTrue(storage.exists(metadataTablePath));
metaClient = HoodieTableMetaClient.reload(metaClient);
assertTrue(metaClient.getTableConfig().isMetadataTableAvailable());
// File groups should be created as in the config
HoodieBackedTableMetadata metadataReader = (HoodieBackedTableMetadata) metadata(client);
- assertEquals(HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataReader.getMetadataMetaClient(), Option.empty(),
+ assertEquals(HoodieTableMetadataUtil.getPartitionLatestFileSlices(
+ metadataReader.getMetadataMetaClient(), Option.empty(),
MetadataPartitionType.FILES.getPartitionPath()).size(), 1);
- assertEquals(HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataReader.getMetadataMetaClient(), Option.empty(),
+ assertEquals(HoodieTableMetadataUtil.getPartitionLatestFileSlices(
+ metadataReader.getMetadataMetaClient(), Option.empty(),
MetadataPartitionType.RECORD_INDEX.getPartitionPath()).size(), 5);
}
// remove the MDT partition from dataset to simulate failed bootstrap
Properties updateProperties = new Properties();
updateProperties.setProperty(HoodieTableConfig.TABLE_METADATA_PARTITIONS.key(), "");
- HoodieTableConfig.update(fs, new Path(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME),
+ HoodieTableConfig.update(storage,
+ new StoragePath(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME),
updateProperties);
metaClient = HoodieTableMetaClient.reload(metaClient);
@@ -1354,7 +1383,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
this.tableType = COPY_ON_WRITE;
initPath();
initFileSystem(basePath, hadoopConf);
- fs.mkdirs(new Path(basePath));
+ storage.createDirectory(new StoragePath(basePath));
initMetaClient(tableType);
initTestDataGenerator();
metadataTableBasePath = getMetadataTableBasePath(basePath);
@@ -1503,9 +1532,11 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
writeStatuses = client.insert(records, commit2);
assertNoWriteErrors(writeStatuses);
// remove latest completed delta commit from MDT.
- Path toDelete = HoodieTestUtils
- .getCompleteInstantPath(metaClient.getFs(), new Path(metaClient.getMetaPath() + "/metadata/.hoodie/"), commit2, HoodieTimeline.DELTA_COMMIT_ACTION);
- metaClient.getFs().delete(toDelete);
+ StoragePath toDelete = HoodieTestUtils.getCompleteInstantPath(
+ metaClient.getStorage(),
+ new StoragePath(metaClient.getMetaPath() + "/metadata" + "/" + ".hoodie/"),
+ commit2, HoodieTimeline.DELTA_COMMIT_ACTION);
+ metaClient.getStorage().deleteDirectory(toDelete);
// Write 3 (updates)
client.close();
@@ -1517,16 +1548,24 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
assertNoWriteErrors(writeStatuses);
// ensure that 000003 is after rollback of the partially failed 2nd commit.
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setBasePath(metaClient.getMetaPath() + "/metadata/").setConf(metaClient.getHadoopConf()).build();
- HoodieInstant rollbackInstant = metadataMetaClient.getActiveTimeline().getRollbackTimeline().getInstants().get(0);
+ HoodieTableMetaClient metadataMetaClient =
+ HoodieTableMetaClient.builder().setBasePath(metaClient.getMetaPath() + "/metadata/")
+ .setConf(metaClient.getHadoopConf()).build();
+ HoodieInstant rollbackInstant =
+ metadataMetaClient.getActiveTimeline().getRollbackTimeline().getInstants().get(0);
// collect all commit meta files from metadata table.
- FileStatus[] metaFiles = metaClient.getFs().listStatus(new Path(metaClient.getMetaPath() + "/metadata/.hoodie"));
- List<FileStatus> commit3Files = Arrays.stream(metaFiles).filter(fileStatus ->
- fileStatus.getPath().getName().contains(commit3)
- && fileStatus.getPath().getName().contains(HoodieTimeline.DELTA_COMMIT_ACTION)).collect(Collectors.toList());
- List<FileStatus> rollbackFiles = Arrays.stream(metaFiles).filter(fileStatus ->
- fileStatus.getPath().getName().equals(rollbackInstant.getFileName())).collect(Collectors.toList());
+ List<StoragePathInfo> metaFiles = metaClient.getStorage().listDirectEntries(
+ new StoragePath(metaClient.getMetaPath() + "/metadata/.hoodie"));
+ List<StoragePathInfo> commit3Files = metaFiles.stream()
+ .filter(pathInfo ->
+ pathInfo.getPath().getName().contains(commit3)
+ && pathInfo.getPath().getName().contains(HoodieTimeline.DELTA_COMMIT_ACTION))
+ .collect(Collectors.toList());
+ List<StoragePathInfo> rollbackFiles = metaFiles.stream()
+ .filter(fileStatus ->
+ fileStatus.getPath().getName().equals(rollbackInstant.getFileName()))
+ .collect(Collectors.toList());
// ensure commit3's delta commit in MDT has last mod time > the actual rollback for previous failed commit i.e. commit2.
// if rollback wasn't eager, rollback's last mod time will be not larger than the commit3'd delta commit last mod time.
@@ -2100,8 +2139,9 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
}
// Metadata table should have been bootstrapped
- assertTrue(fs.exists(new Path(metadataTableBasePath)), "Metadata table should exist");
- FileStatus oldStatus = fs.getFileStatus(new Path(metadataTableBasePath));
+ assertTrue(storage.exists(new StoragePath(metadataTableBasePath)),
+ "Metadata table should exist");
+ StoragePathInfo oldInfo = storage.getPathInfo(new StoragePath(metadataTableBasePath));
// trigger partial commit
metaClient.reloadActiveTimeline();
@@ -2133,10 +2173,12 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
}
initMetaClient();
- assertEquals(metaClient.getTableConfig().getTableVersion().versionCode(), HoodieTableVersion.current().versionCode());
- assertTrue(fs.exists(new Path(metadataTableBasePath)), "Metadata table should exist");
- FileStatus newStatus = fs.getFileStatus(new Path(metadataTableBasePath));
- assertTrue(oldStatus.getModificationTime() < newStatus.getModificationTime());
+ assertEquals(metaClient.getTableConfig().getTableVersion().versionCode(),
+ HoodieTableVersion.current().versionCode());
+ assertTrue(storage.exists(new StoragePath(metadataTableBasePath)),
+ "Metadata table should exist");
+ StoragePathInfo newInfo = storage.getPathInfo(new StoragePath(metadataTableBasePath));
+ assertTrue(oldInfo.getModificationTime() < newInfo.getModificationTime());
}
/**
@@ -2173,9 +2215,10 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// There is no way to simulate failed commit on the main dataset, hence we simply delete the completed
// instant so that only the inflight is left over.
- String commitInstantFileName = metaClient.getActiveTimeline().getReverseOrderedInstants().findFirst().get().getFileName();
- assertTrue(fs.delete(new Path(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME,
- commitInstantFileName), false));
+ String commitInstantFileName =
+ metaClient.getActiveTimeline().getReverseOrderedInstants().findFirst().get().getFileName();
+ assertTrue(storage.deleteFile(
+ new StoragePath(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME, commitInstantFileName)));
}
try (HoodieJavaWriteClient client = new HoodieJavaWriteClient(engineContext,
@@ -2210,11 +2253,13 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
validateMetadata(client);
}
- final Path metadataTablePath = new Path(getMetadataTableBasePath(writeConfig.getBasePath()));
- assertTrue(fs.exists(metadataTablePath), "metadata table should exist.");
+ final StoragePath metadataTablePath =
+ new StoragePath(getMetadataTableBasePath(writeConfig.getBasePath()));
+ assertTrue(storage.exists(metadataTablePath), "metadata table should exist.");
deleteMetadataTable(metaClient, context, false);
- assertFalse(fs.exists(metadataTablePath), "metadata table should not exist after being deleted.");
+ assertFalse(storage.exists(metadataTablePath),
+ "metadata table should not exist after being deleted.");
writeConfig = getWriteConfigBuilder(true, true, false).build();
try (HoodieJavaWriteClient client = new HoodieJavaWriteClient(engineContext, writeConfig)) {
@@ -2227,7 +2272,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
}
// Metadata table is recreated, during bootstrapping of metadata table.
- assertTrue(fs.exists(metadataTablePath));
+ assertTrue(storage.exists(metadataTablePath));
}
/**
@@ -2273,9 +2318,10 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// There is no way to simulate failed commit on the main dataset, hence we simply delete the completed
// instant so that only the inflight is left over.
- String commitInstantFileName = metaClient.getActiveTimeline().getReverseOrderedInstants().findFirst().get().getFileName();
- assertTrue(fs.delete(new Path(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME,
- commitInstantFileName), false));
+ String commitInstantFileName =
+ metaClient.getActiveTimeline().getReverseOrderedInstants().findFirst().get().getFileName();
+ assertTrue(storage.deleteFile(
+ new StoragePath(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME, commitInstantFileName)));
}
try (HoodieJavaWriteClient client = new HoodieJavaWriteClient(engineContext,
@@ -2410,17 +2456,24 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// 1 partition should be cleaned
assertEquals(cleanMetadata.getPartitionMetadata().size(), 1);
// 1 file cleaned
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getSuccessDeleteFiles().size(), 1);
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getFailedDeleteFiles().size(), 0);
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getDeletePathPatterns().size(), 1);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getSuccessDeleteFiles().size(), 1);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getFailedDeleteFiles().size(), 0);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getDeletePathPatterns().size(), 1);
// To simulate failed clean on the main dataset, we will delete the completed clean instant
- String cleanInstantFileName = metaClient.reloadActiveTimeline().getCleanerTimeline().filterCompletedInstants()
- .getReverseOrderedInstants().findFirst().get().getFileName();
- assertTrue(fs.delete(new Path(basePath + StoragePath.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME,
- cleanInstantFileName), false));
- assertEquals(metaClient.reloadActiveTimeline().getCleanerTimeline().filterInflights().countInstants(), 1);
- assertEquals(metaClient.reloadActiveTimeline().getCleanerTimeline().filterCompletedInstants().countInstants(), 0);
+ String cleanInstantFileName =
+ metaClient.reloadActiveTimeline().getCleanerTimeline().filterCompletedInstants()
+ .getReverseOrderedInstants().findFirst().get().getFileName();
+ assertTrue(storage.deleteFile(new StoragePath(
+ basePath + StoragePath.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME, cleanInstantFileName)));
+ assertEquals(
+ metaClient.reloadActiveTimeline().getCleanerTimeline().filterInflights().countInstants(),
+ 1);
+ assertEquals(metaClient.reloadActiveTimeline().getCleanerTimeline().filterCompletedInstants()
+ .countInstants(), 0);
// Initiate another clean. The previous leftover clean will be attempted and no other clean will be scheduled.
String newCleanInstantTime = "00" + index++;
@@ -2429,9 +2482,12 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// 1 partition should be cleaned
assertEquals(cleanMetadata.getPartitionMetadata().size(), 1);
// 1 file cleaned but was already deleted so will be a failed delete
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getSuccessDeleteFiles().size(), 0);
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getFailedDeleteFiles().size(), 1);
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getDeletePathPatterns().size(), 1);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getSuccessDeleteFiles().size(), 0);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getFailedDeleteFiles().size(), 1);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getDeletePathPatterns().size(), 1);
validateMetadata(client);
}
@@ -2665,39 +2721,46 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// Files within each partition should match
HoodieTable table = HoodieJavaTable.create(config, engineContext);
TableFileSystemView tableView = table.getHoodieView();
- List<String> fullPartitionPaths = fsPartitions.stream().map(partition -> basePath + "/" + partition).collect(Collectors.toList());
- Map<String, FileStatus[]> partitionToFilesMap = tableMetadata.getAllFilesInPartitions(fullPartitionPaths);
+ List<String> fullPartitionPaths =
+ fsPartitions.stream().map(partition -> basePath + "/" + partition)
+ .collect(Collectors.toList());
+ Map<String, List<StoragePathInfo>> partitionToFilesMap =
+ tableMetadata.getAllFilesInPartitions(fullPartitionPaths);
assertEquals(fsPartitions.size(), partitionToFilesMap.size());
fsPartitions.forEach(partition -> {
try {
- Path partitionPath;
+ StoragePath partitionPath;
if (partition.equals("")) {
// Should be the non-partitioned case
- partitionPath = new Path(basePath);
+ partitionPath = new StoragePath(basePath);
} else {
- partitionPath = new Path(basePath, partition);
+ partitionPath = new StoragePath(basePath, partition);
}
- FileStatus[] fsStatuses = FSUtils.getAllDataFilesInPartition(fs, partitionPath);
+ List<StoragePathInfo> allFilesList =
+ FSUtils.getAllDataFilesInPartition(storage, partitionPath);
if (ignoreFilesWithCommit.isPresent()) {
- fsStatuses = Arrays.stream(fsStatuses).filter(fileStatus -> !fileStatus.getPath().getName().contains(ignoreFilesWithCommit.get()))
- .collect(Collectors.toList()).toArray(new FileStatus[0]);
+ allFilesList = allFilesList.stream()
+ .filter(fileStatus -> !fileStatus.getPath().getName()
+ .contains(ignoreFilesWithCommit.get()))
+ .collect(Collectors.toList());
}
- FileStatus[] metaStatuses = tableMetadata.getAllFilesInPartition(partitionPath);
- List<String> fsFileNames = Arrays.stream(fsStatuses)
+ List<StoragePathInfo> metaFilesList = tableMetadata.getAllFilesInPartition(partitionPath);
+ List<String> fsFileNames = allFilesList.stream()
.map(s -> s.getPath().getName()).collect(Collectors.toList());
- List<String> metadataFilenames = Arrays.stream(metaStatuses)
+ List<String> metadataFilenames = metaFilesList.stream()
.map(s -> s.getPath().getName()).collect(Collectors.toList());
Collections.sort(fsFileNames);
Collections.sort(metadataFilenames);
- assertEquals(fsStatuses.length, partitionToFilesMap.get(partitionPath.toString()).length);
+ assertEquals(allFilesList.size(), partitionToFilesMap.get(partitionPath.toString()).size());
// File sizes should be valid
- Arrays.stream(metaStatuses).forEach(s -> assertTrue(s.getLen() > 0));
+ metaFilesList.stream().forEach(s -> assertTrue(s.getLength() > 0));
- if ((fsFileNames.size() != metadataFilenames.size()) || (!fsFileNames.equals(metadataFilenames))) {
+ if ((fsFileNames.size() != metadataFilenames.size())
+ || (!fsFileNames.equals(metadataFilenames))) {
LOG.info("*** File system listing = " + Arrays.toString(fsFileNames.toArray()));
LOG.info("*** Metadata listing = " + Arrays.toString(metadataFilenames.toArray()));
@@ -2713,27 +2776,27 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
}
}
- // Block sizes should be valid
- Arrays.stream(metaStatuses).forEach(s -> assertTrue(s.getBlockSize() > 0));
- List<Long> fsBlockSizes = Arrays.stream(fsStatuses).map(FileStatus::getBlockSize).collect(Collectors.toList());
- Collections.sort(fsBlockSizes);
- List<Long> metadataBlockSizes = Arrays.stream(metaStatuses).map(FileStatus::getBlockSize).collect(Collectors.toList());
- Collections.sort(metadataBlockSizes);
- assertEquals(fsBlockSizes, metadataBlockSizes);
-
- assertEquals(fsFileNames.size(), metadataFilenames.size(), "Files within partition " + partition + " should match");
- assertTrue(fsFileNames.equals(metadataFilenames), "Files within partition " + partition + " should match");
+ assertEquals(fsFileNames.size(), metadataFilenames.size(),
+ "Files within partition " + partition + " should match");
+ assertTrue(fsFileNames.equals(metadataFilenames),
+ "Files within partition " + partition + " should match");
// FileSystemView should expose the same data
- List<HoodieFileGroup> fileGroups = tableView.getAllFileGroups(partition).collect(Collectors.toList());
- fileGroups.addAll(tableView.getAllReplacedFileGroups(partition).collect(Collectors.toList()));
-
- fileGroups.forEach(g -> LoggerFactory.getLogger(TestJavaHoodieBackedMetadata.class).info(g.toString()));
- fileGroups.forEach(g -> g.getAllBaseFiles().forEach(b -> LoggerFactory.getLogger(TestJavaHoodieBackedMetadata.class).info(b.toString())));
- fileGroups.forEach(g -> g.getAllFileSlices().forEach(s -> LoggerFactory.getLogger(TestJavaHoodieBackedMetadata.class).info(s.toString())));
+ List<HoodieFileGroup> fileGroups =
+ tableView.getAllFileGroups(partition).collect(Collectors.toList());
+ fileGroups.addAll(
+ tableView.getAllReplacedFileGroups(partition).collect(Collectors.toList()));
+
+ fileGroups.forEach(
+ g -> LoggerFactory.getLogger(TestJavaHoodieBackedMetadata.class).info(g.toString()));
+ fileGroups.forEach(g -> g.getAllBaseFiles().forEach(
+ b -> LoggerFactory.getLogger(TestJavaHoodieBackedMetadata.class).info(b.toString())));
+ fileGroups.forEach(g -> g.getAllFileSlices().forEach(
+ s -> LoggerFactory.getLogger(TestJavaHoodieBackedMetadata.class).info(s.toString())));
long numFiles = fileGroups.stream()
- .mapToLong(g -> g.getAllBaseFiles().count() + g.getAllFileSlices().mapToLong(s -> s.getLogFiles().count()).sum())
+ .mapToLong(g -> g.getAllBaseFiles().count()
+ + g.getAllFileSlices().mapToLong(s -> s.getLogFiles().count()).sum())
.sum();
assertEquals(metadataFilenames.size(), numFiles);
} catch (IOException e) {
@@ -2792,25 +2855,32 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
private void verifyMetadataColumnStatsRecords(List<HoodieLogFile> logFiles) throws IOException {
for (HoodieLogFile logFile : logFiles) {
- FileStatus[] fsStatus = fs.listStatus(logFile.getPath());
- MessageType writerSchemaMsg = TableSchemaResolver.readSchemaFromLogFile(fs, logFile.getPath());
+ List<StoragePathInfo> pathInfoList = storage.listDirectEntries(logFile.getPath());
+ MessageType writerSchemaMsg = TableSchemaResolver.readSchemaFromLogFile(storage,
+ logFile.getPath());
if (writerSchemaMsg == null) {
// not a data block
continue;
}
Schema writerSchema = new AvroSchemaConverter().convert(writerSchemaMsg);
- try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(fs, new HoodieLogFile(fsStatus[0].getPath()), writerSchema)) {
+ try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(storage,
+ new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) {
while (logFileReader.hasNext()) {
HoodieLogBlock logBlock = logFileReader.next();
if (logBlock instanceof HoodieDataBlock) {
- try (ClosableIterator<HoodieRecord<IndexedRecord>> recordItr = ((HoodieDataBlock) logBlock).getRecordIterator(HoodieRecordType.AVRO)) {
+ try (
+ ClosableIterator<HoodieRecord<IndexedRecord>> recordItr = ((HoodieDataBlock) logBlock).getRecordIterator(
+ HoodieRecordType.AVRO)) {
recordItr.forEachRemaining(indexRecord -> {
final GenericRecord record = (GenericRecord) indexRecord.getData();
- final GenericRecord colStatsRecord = (GenericRecord) record.get(HoodieMetadataPayload.SCHEMA_FIELD_ID_COLUMN_STATS);
+ final GenericRecord colStatsRecord =
+ (GenericRecord) record.get(HoodieMetadataPayload.SCHEMA_FIELD_ID_COLUMN_STATS);
assertNotNull(colStatsRecord);
- assertNotNull(colStatsRecord.get(HoodieMetadataPayload.COLUMN_STATS_FIELD_COLUMN_NAME));
- assertNotNull(colStatsRecord.get(HoodieMetadataPayload.COLUMN_STATS_FIELD_NULL_COUNT));
+ assertNotNull(
+ colStatsRecord.get(HoodieMetadataPayload.COLUMN_STATS_FIELD_COLUMN_NAME));
+ assertNotNull(
+ colStatsRecord.get(HoodieMetadataPayload.COLUMN_STATS_FIELD_NULL_COUNT));
/**
* TODO: some types of field may have null min/max as these statistics are only supported for primitive types
* assertNotNull(colStatsRecord.get(HoodieMetadataPayload.COLUMN_STATS_FIELD_MAX_VALUE));
@@ -2827,11 +2897,12 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
/**
* Returns the list of all files in the dataset by iterating over the metadata table.
*/
- private List<Path> getAllFiles(HoodieTableMetadata metadata) throws Exception {
- List<Path> allfiles = new ArrayList<>();
+ private List<StoragePath> getAllFiles(HoodieTableMetadata metadata) throws Exception {
+ List<StoragePath> allfiles = new ArrayList<>();
for (String partition : metadata.getAllPartitionPaths()) {
- for (FileStatus status : metadata.getAllFilesInPartition(new Path(basePath, partition))) {
- allfiles.add(status.getPath());
+ for (StoragePathInfo pathInfo : metadata.getAllFilesInPartition(
+ new StoragePath(basePath, partition))) {
+ allfiles.add(pathInfo.getPath());
}
}
@@ -2849,8 +2920,10 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
private void changeTableVersion(HoodieTableVersion version) throws IOException {
metaClient = HoodieTableMetaClient.reload(metaClient);
metaClient.getTableConfig().setTableVersion(version);
- Path propertyFile = new Path(metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE);
- try (OutputStream os = metaClient.getFs().create(propertyFile)) {
+ StoragePath propertyFile =
+ new StoragePath(
+ metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE);
+ try (OutputStream os = metaClient.getStorage().create(propertyFile)) {
metaClient.getTableConfig().getProps().store(os, "");
}
}
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
index 9749eb55e1e..11fc62d67ff 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
@@ -87,6 +87,8 @@ import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.KeyGenerator;
import org.apache.hudi.keygen.factory.HoodieAvroKeyGeneratorFactory;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.table.HoodieJavaTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.HoodieWriteMetadata;
@@ -95,7 +97,7 @@ import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.hudi.testutils.HoodieJavaClientTestHarness;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.BeforeEach;
@@ -502,7 +504,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
for (int i = 0; i < fullPartitionPaths.length; i++) {
fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
}
- assertEquals(200, countRowsInPaths(basePath, fs, fullPartitionPaths),
+ assertEquals(200, countRowsInPaths(basePath, storage, fullPartitionPaths),
"Must contain " + 200 + " records");
// Perform Delete again on upgraded dataset.
@@ -796,18 +798,24 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
assertNoWriteErrors(statuses);
metaClient = HoodieTableMetaClient.reload(metaClient);
- HoodieInstant replaceCommitInstant = metaClient.getActiveTimeline().getCompletedReplaceTimeline().firstInstant().get();
+ HoodieInstant replaceCommitInstant =
+ metaClient.getActiveTimeline().getCompletedReplaceTimeline().firstInstant().get();
HoodieReplaceCommitMetadata replaceCommitMetadata = HoodieReplaceCommitMetadata
- .fromBytes(metaClient.getActiveTimeline().getInstantDetails(replaceCommitInstant).get(), HoodieReplaceCommitMetadata.class);
+ .fromBytes(metaClient.getActiveTimeline().getInstantDetails(replaceCommitInstant).get(),
+ HoodieReplaceCommitMetadata.class);
List<String> filesFromReplaceCommit = new ArrayList<>();
replaceCommitMetadata.getPartitionToWriteStats()
.forEach((k, v) -> v.forEach(entry -> filesFromReplaceCommit.add(entry.getPath())));
// find all parquet files created as part of clustering. Verify it matches w/ what is found in replace commit metadata.
- FileStatus[] fileStatuses = fs.listStatus(new Path(basePath + "/" + partitionPath));
- List<String> clusteredFiles = Arrays.stream(fileStatuses).filter(entry -> entry.getPath().getName().contains(replaceCommitInstant.getTimestamp()))
- .map(fileStatus -> partitionPath + "/" + fileStatus.getPath().getName()).collect(Collectors.toList());
+ List<StoragePathInfo> pathInfoList =
+ storage.listDirectEntries(new StoragePath(basePath + "/" + partitionPath));
+ List<String> clusteredFiles = pathInfoList.stream()
+ .filter(
+ entry -> entry.getPath().getName().contains(replaceCommitInstant.getTimestamp()))
+ .map(fileStatus -> partitionPath + "/" + fileStatus.getPath().getName())
+ .collect(Collectors.toList());
assertEquals(clusteredFiles, filesFromReplaceCommit);
}
}
@@ -1020,7 +1028,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
@NotNull
private Set<String> verifyRecordKeys(List<HoodieRecord> expectedRecords, List<WriteStatus> allStatus, List<GenericRecord> records) {
for (WriteStatus status : allStatus) {
- Path filePath = new Path(basePath, status.getStat().getPath());
+ StoragePath filePath = new StoragePath(basePath, status.getStat().getPath());
records.addAll(getFileUtilsInstance(metaClient).readAvroRecords(hadoopConf, filePath));
}
Set<String> expectedKeys = recordsToRecordKeySet(expectedRecords);
@@ -1079,7 +1087,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
}
assertEquals(expectedTotalRecords,
- countRowsInPaths(basePath, fs, fullPartitionPaths),
+ countRowsInPaths(basePath, storage, fullPartitionPaths),
"Must contain " + expectedTotalRecords + " records");
return Pair.of(keys, inserts);
}
@@ -1134,15 +1142,18 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
HoodieInstant commitInstant = new HoodieInstant(false, actionType, instantTime);
HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants();
HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
- .fromBytes(commitTimeline.getInstantDetails(commitInstant).get(), HoodieCommitMetadata.class);
+ .fromBytes(commitTimeline.getInstantDetails(commitInstant).get(),
+ HoodieCommitMetadata.class);
String basePath = table.getMetaClient().getBasePath();
- Collection<String> commitPathNames = commitMetadata.getFileIdAndFullPaths(new Path(basePath)).values();
+ Collection<String> commitPathNames =
+ commitMetadata.getFileIdAndFullPaths(new StoragePath(basePath)).values();
// Read from commit file
HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(
- metaClient.reloadActiveTimeline().getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime)).get(),
+ metaClient.reloadActiveTimeline()
+ .getInstantDetails(new HoodieInstant(COMPLETED, COMMIT_ACTION, instantTime)).get(),
HoodieCommitMetadata.class);
- HashMap<String, String> paths = metadata.getFileIdAndFullPaths(new Path(basePath));
+ HashMap<String, String> paths = metadata.getFileIdAndFullPaths(new StoragePath(basePath));
// Compare values in both to make sure they are equal.
for (String pathName : paths.values()) {
assertTrue(commitPathNames.contains(pathName));
@@ -1220,22 +1231,24 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
.withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build();
HoodieJavaWriteClient client = getHoodieWriteClient(cfg);
- Pair<Path, List<WriteStatus>> result = testConsistencyCheck(metaClient, instantTime, enableOptimisticConsistencyGuard);
+ Pair<StoragePath, List<WriteStatus>> result = testConsistencyCheck(
+ metaClient, instantTime, enableOptimisticConsistencyGuard);
// Delete orphan marker and commit should succeed
- metaClient.getFs().delete(result.getKey(), false);
+ metaClient.getStorage().deleteFile(result.getKey());
if (!enableOptimisticConsistencyGuard) {
assertTrue(client.commit(instantTime, result.getRight()), "Commit should succeed");
assertTrue(testTable.commitExists(instantTime),
"After explicit commit, commit file should be created");
// Marker directory must be removed
- assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
+ assertFalse(metaClient.getStorage()
+ .exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
} else {
// with optimistic, first client.commit should have succeeded.
assertTrue(testTable.commitExists(instantTime),
"After explicit commit, commit file should be created");
// Marker directory must be removed
- assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
+ assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
}
}
@@ -1267,13 +1280,13 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
assertFalse(testTable.commitExists(instantTime),
"After explicit rollback, commit file should not be present");
// Marker directory must be removed after rollback
- assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
+ assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
} else {
// if optimistic CG is enabled, commit should have succeeded.
assertTrue(testTable.commitExists(instantTime),
"With optimistic CG, first commit should succeed. commit file should be present");
// Marker directory must be removed after rollback
- assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
+ assertFalse(metaClient.getStorage().exists(new StoragePath(metaClient.getMarkerFolderPath(instantTime))));
client.rollback(instantTime);
assertFalse(testTable.commitExists(instantTime),
"After explicit rollback, commit file should not be present");
@@ -1492,7 +1505,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
service.shutdown();
}
- private Pair<Path, List<WriteStatus>> testConsistencyCheck(HoodieTableMetaClient metaClient, String instantTime, boolean enableOptimisticConsistencyGuard)
+ private Pair<StoragePath, List<WriteStatus>> testConsistencyCheck(HoodieTableMetaClient metaClient, String instantTime, boolean enableOptimisticConsistencyGuard)
throws Exception {
HoodieWriteConfig cfg = !enableOptimisticConsistencyGuard ? (getConfigBuilder().withAutoCommit(false)
.withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true)
@@ -1512,9 +1525,10 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
// This should fail the commit
String partitionPath;
String markerFolderPath = metaClient.getMarkerFolderPath(instantTime);
+ FileSystem fs = (FileSystem) storage.getFileSystem();
if (cfg.getMarkersType() == MarkerType.TIMELINE_SERVER_BASED) {
String markerName = MarkerUtils.readTimelineServerBasedMarkersFromFileSystem(
- markerFolderPath, fs, context, 1).values().stream()
+ markerFolderPath, storage, context, 1).values().stream()
.flatMap(Collection::stream).findFirst().get();
partitionPath = new Path(markerFolderPath, markerName).getParent().toString();
} else {
@@ -1524,7 +1538,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
.limit(1).map(status -> status.getPath().getParent().toString()).collect(Collectors.toList()).get(0);
}
- Option<Path> markerFilePath = WriteMarkersFactory.get(
+ Option<StoragePath> markerFilePath = WriteMarkersFactory.get(
cfg.getMarkersType(), getHoodieTable(metaClient, cfg), instantTime)
.create(partitionPath,
FSUtils.makeBaseFileName(instantTime, "1-0-1", UUID.randomUUID().toString(), HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension()),
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
index 87e690bfca0..334ba9f862d 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
@@ -41,6 +41,7 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hadoop.HoodieParquetInputFormat;
import org.apache.hudi.hadoop.utils.HoodieHiveUtils;
import org.apache.hudi.io.HoodieCreateHandle;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieJavaCopyOnWriteTable;
import org.apache.hudi.table.HoodieJavaTable;
import org.apache.hudi.table.HoodieTable;
@@ -92,7 +93,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTable table = HoodieJavaTable.create(config, context, metaClient);
- Pair<Path, String> newPathWithWriteToken = Arrays.asList(1).stream().map(x -> {
+ Pair<StoragePath, String> newPathWithWriteToken = Arrays.asList(1).stream().map(x -> {
HoodieRecord record = mock(HoodieRecord.class);
when(record.getPartitionPath()).thenReturn(partitionPath);
String writeToken = FSUtils.makeWriteToken(context.getTaskContextSupplier().getPartitionIdSupplier().get(),
@@ -100,7 +101,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
context.getTaskContextSupplier().getAttemptIdSupplier().get());
HoodieCreateHandle io = new HoodieCreateHandle(config, instantTime, table, partitionPath, fileName,
context.getTaskContextSupplier());
- Pair<Path, String> result = Pair.of(io.makeNewPath(record.getPartitionPath()), writeToken);
+ Pair<StoragePath, String> result = Pair.of(io.makeNewPath(record.getPartitionPath()), writeToken);
io.close();
return result;
}).collect(Collectors.toList()).get(0);
@@ -160,13 +161,13 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(hadoopConf, filePath);
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(hadoopConf, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(hadoopConf, filePath);
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(hadoopConf, new StoragePath(filePath.toUri()));
GenericRecord newRecord;
int index = 0;
for (GenericRecord record : fileRecords) {
@@ -201,7 +202,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
// Check whether the record has been updated
Path updatedFilePath = allFiles[0].getPath();
BloomFilter updatedFilter =
- fileUtils.readBloomFilterFromMetadata(hadoopConf, updatedFilePath);
+ fileUtils.readBloomFilterFromMetadata(hadoopConf, new StoragePath(updatedFilePath.toUri()));
for (HoodieRecord record : records) {
// No change to the _row_key
assertTrue(updatedFilter.mightContain(record.getRecordKey()));
@@ -432,7 +433,9 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
WriteStatus writeStatus = ws.get(0).get(0);
String fileId = writeStatus.getFileId();
- metaClient.getFs().create(new Path(Paths.get(basePath, ".hoodie", "000.commit").toString())).close();
+ metaClient.getStorage()
+ .create(new StoragePath(Paths.get(basePath, ".hoodie", "000.commit").toString()))
+ .close();
//TODO : Find race condition that causes the timeline sometime to reflect 000.commit and sometimes not
final HoodieJavaCopyOnWriteTable reloadedTable = (HoodieJavaCopyOnWriteTable) HoodieJavaTable.create(config, context, HoodieTableMetaClient.reload(metaClient));
@@ -505,13 +508,13 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
// Read out the bloom filter and make sure filter can answer record exist or not
Path filePath = allFiles[0].getPath();
- BloomFilter filter = fileUtils.readBloomFilterFromMetadata(hadoopConf, filePath);
+ BloomFilter filter = fileUtils.readBloomFilterFromMetadata(hadoopConf, new StoragePath(filePath.toUri()));
for (HoodieRecord record : records) {
assertTrue(filter.mightContain(record.getRecordKey()));
}
// Read the base file, check the record content
- List<GenericRecord> fileRecords = fileUtils.readAvroRecords(hadoopConf, filePath);
+ List<GenericRecord> fileRecords = fileUtils.readAvroRecords(hadoopConf, new StoragePath(filePath.toUri()));
int index = 0;
for (GenericRecord record : fileRecords) {
assertEquals(records.get(index).getRecordKey(), record.get("_row_key").toString());
@@ -530,7 +533,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
filePath = allFiles[0].getPath();
// Read the base file, check the record content
- fileRecords = fileUtils.readAvroRecords(hadoopConf, filePath);
+ fileRecords = fileUtils.readAvroRecords(hadoopConf, new StoragePath(filePath.toUri()));
// Check that the two records are deleted successfully
assertEquals(1, fileRecords.size());
assertEquals(records.get(1).getRecordKey(), fileRecords.get(0).get("_row_key").toString());
@@ -547,7 +550,7 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
filePath = allFiles[0].getPath();
// Read the base file, check the record content
- fileRecords = fileUtils.readAvroRecords(hadoopConf, filePath);
+ fileRecords = fileUtils.readAvroRecords(hadoopConf, new StoragePath(filePath.toUri()));
// Check whether all records have been deleted
assertEquals(0, fileRecords.size());
}
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
index cddf4561e70..96ac7444eca 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
@@ -60,7 +60,6 @@ import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieMetadataException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.index.JavaHoodieIndexFactory;
import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
@@ -69,6 +68,10 @@ import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.metadata.JavaHoodieBackedTableMetadataWriter;
import org.apache.hudi.metadata.MetadataPartitionType;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.table.HoodieJavaTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.utils.HoodieWriterClientTestHarness;
@@ -78,7 +81,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -119,7 +121,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
protected Configuration hadoopConf;
protected HoodieJavaEngineContext context;
protected TestJavaTaskContextSupplier taskContextSupplier;
- protected FileSystem fs;
+ protected HoodieStorage storage;
protected ExecutorService executorService;
protected HoodieTableFileSystemView tableView;
protected HoodieJavaWriteClient writeClient;
@@ -188,9 +190,9 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
throw new IllegalStateException("The base path has not been initialized.");
}
- fs = HadoopFSUtils.getFs(basePath, hadoopConf);
- if (fs instanceof LocalFileSystem) {
- LocalFileSystem lfs = (LocalFileSystem) fs;
+ storage = HoodieStorageUtils.getStorage(basePath, hadoopConf);
+ if (storage.getFileSystem() instanceof LocalFileSystem) {
+ LocalFileSystem lfs = (LocalFileSystem) storage.getFileSystem();
// With LocalFileSystem, with checksum disabled, fs.open() returns an inputStream which is FSInputStream
// This causes ClassCastExceptions in LogRecordScanner (and potentially other places) calling fs.open
// So, for the tests, we enforce checksum verification to circumvent the problem
@@ -199,10 +201,10 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
}
protected void cleanupFileSystem() throws IOException {
- if (fs != null) {
- LOG.warn("Closing file-system instance used in previous test-run");
- fs.close();
- fs = null;
+ if (storage != null) {
+ LOG.warn("Closing HoodieStorage instance used in previous test-run");
+ storage.close();
+ storage = null;
}
}
@@ -303,13 +305,17 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTable table = HoodieJavaTable.create(writeConfig, engineContext);
TableFileSystemView tableView = table.getHoodieView();
- List<String> fullPartitionPaths = fsPartitions.stream().map(partition -> basePath + "/" + partition).collect(Collectors.toList());
- Map<String, FileStatus[]> partitionToFilesMap = tableMetadata.getAllFilesInPartitions(fullPartitionPaths);
+ List<String> fullPartitionPaths =
+ fsPartitions.stream().map(partition -> basePath + "/" + partition)
+ .collect(Collectors.toList());
+ Map<String, List<StoragePathInfo>> partitionToFilesMap =
+ tableMetadata.getAllFilesInPartitions(fullPartitionPaths);
assertEquals(fsPartitions.size(), partitionToFilesMap.size());
fsPartitions.forEach(partition -> {
try {
- validateFilesPerPartition(testTable, tableMetadata, tableView, partitionToFilesMap, partition);
+ validateFilesPerPartition(testTable, tableMetadata, tableView, partitionToFilesMap,
+ partition);
} catch (IOException e) {
fail("Exception should not be raised: " + e);
}
@@ -321,47 +327,56 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
LOG.info("Validation time=" + timer.endTimer());
}
- protected void validateFilesPerPartition(HoodieTestTable testTable, HoodieTableMetadata tableMetadata, TableFileSystemView tableView,
- Map<String, FileStatus[]> partitionToFilesMap, String partition) throws IOException {
- Path partitionPath;
+ protected void validateFilesPerPartition(HoodieTestTable testTable,
+ HoodieTableMetadata tableMetadata,
+ TableFileSystemView tableView,
+ Map<String, List<StoragePathInfo>> partitionToFilesMap,
+ String partition) throws IOException {
+ StoragePath partitionPath;
if (partition.equals("")) {
// Should be the non-partitioned case
- partitionPath = new Path(basePath);
+ partitionPath = new StoragePath(basePath);
} else {
- partitionPath = new Path(basePath, partition);
+ partitionPath = new StoragePath(basePath, partition);
}
FileStatus[] fsStatuses = testTable.listAllFilesInPartition(partition);
- FileStatus[] metaStatuses = tableMetadata.getAllFilesInPartition(partitionPath);
+ List<StoragePathInfo> metaFilesList = tableMetadata.getAllFilesInPartition(partitionPath);
List<String> fsFileNames = Arrays.stream(fsStatuses)
.map(s -> s.getPath().getName()).collect(Collectors.toList());
- List<String> metadataFilenames = Arrays.stream(metaStatuses)
+ List<String> metadataFilenames = metaFilesList.stream()
.map(s -> s.getPath().getName()).collect(Collectors.toList());
Collections.sort(fsFileNames);
Collections.sort(metadataFilenames);
assertLinesMatch(fsFileNames, metadataFilenames);
- assertEquals(fsStatuses.length, partitionToFilesMap.get(partitionPath.toString()).length);
+ assertEquals(fsStatuses.length, partitionToFilesMap.get(partitionPath.toString()).size());
// Block sizes should be valid
- Arrays.stream(metaStatuses).forEach(s -> assertTrue(s.getBlockSize() > 0));
+ metaFilesList.forEach(s -> assertTrue(s.getBlockSize() > 0));
List<Long> fsBlockSizes = Arrays.stream(fsStatuses).map(FileStatus::getBlockSize).sorted().collect(Collectors.toList());
- List<Long> metadataBlockSizes = Arrays.stream(metaStatuses).map(FileStatus::getBlockSize).sorted().collect(Collectors.toList());
+ List<Long> metadataBlockSizes = metaFilesList.stream().map(StoragePathInfo::getBlockSize).sorted().collect(Collectors.toList());
assertEquals(fsBlockSizes, metadataBlockSizes);
- assertEquals(fsFileNames.size(), metadataFilenames.size(), "Files within partition " + partition + " should match");
- assertEquals(fsFileNames, metadataFilenames, "Files within partition " + partition + " should match");
+ assertEquals(fsFileNames.size(), metadataFilenames.size(),
+ "Files within partition " + partition + " should match");
+ assertEquals(fsFileNames, metadataFilenames,
+ "Files within partition " + partition + " should match");
// FileSystemView should expose the same data
- List<HoodieFileGroup> fileGroups = tableView.getAllFileGroups(partition).collect(Collectors.toList());
+ List<HoodieFileGroup> fileGroups =
+ tableView.getAllFileGroups(partition).collect(Collectors.toList());
fileGroups.addAll(tableView.getAllReplacedFileGroups(partition).collect(Collectors.toList()));
fileGroups.forEach(g -> LoggerFactory.getLogger(getClass()).info(g.toString()));
- fileGroups.forEach(g -> g.getAllBaseFiles().forEach(b -> LoggerFactory.getLogger(getClass()).info(b.toString())));
- fileGroups.forEach(g -> g.getAllFileSlices().forEach(s -> LoggerFactory.getLogger(getClass()).info(s.toString())));
+ fileGroups.forEach(g -> g.getAllBaseFiles()
+ .forEach(b -> LoggerFactory.getLogger(getClass()).info(b.toString())));
+ fileGroups.forEach(g -> g.getAllFileSlices()
+ .forEach(s -> LoggerFactory.getLogger(getClass()).info(s.toString())));
long numFiles = fileGroups.stream()
- .mapToLong(g -> g.getAllBaseFiles().count() + g.getAllFileSlices().mapToLong(s -> s.getLogFiles().count()).sum())
+ .mapToLong(g -> g.getAllBaseFiles().count()
+ + g.getAllFileSlices().mapToLong(s -> s.getLogFiles().count()).sum())
.sum();
assertEquals(metadataFilenames.size(), numFiles);
}
@@ -588,7 +603,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
List<HoodieKey> deleteRecords = keyGenFunction.apply(numRecordsInThisCommit);
// check the partition metadata is written out
- assertPartitionMetadataForKeys(basePath, deleteRecords, fs);
+ assertPartitionMetadataForKeys(basePath, deleteRecords, storage);
Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieKey>, String> deleteFn = HoodieJavaWriteClient::delete;
List<WriteStatus> result = deleteFn.apply(client, deleteRecords, newCommitTime);
@@ -676,7 +691,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
client.commit(newCommitTime, result);
}
// check the partition metadata is written out
- assertPartitionMetadataForRecords(basePath, records, fs);
+ assertPartitionMetadataForRecords(basePath, records, storage);
// verify that there is a commit
HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
@@ -697,7 +712,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
for (int i = 0; i < fullPartitionPaths.length; i++) {
fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
}
- assertEquals(expTotalRecords, countRowsInPaths(basePath, fs, fullPartitionPaths),
+ assertEquals(expTotalRecords, countRowsInPaths(basePath, storage, fullPartitionPaths),
"Must contain " + expTotalRecords + " records");
if (filterForCommitTimeWithAssert) {
@@ -872,7 +887,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
for (int i = 0; i < fullPartitionPaths.length; i++) {
fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
}
- assertEquals(expTotalRecords, countRowsInPaths(basePath, fs, fullPartitionPaths),
+ assertEquals(expTotalRecords, countRowsInPaths(basePath, storage, fullPartitionPaths),
"Must contain " + expTotalRecords + " records");
if (filerForCommitTimeWithAssert) {
@@ -896,7 +911,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
HashMap<String, String> paths =
getLatestFileIDsToFullPath(basePath, commitTimeline, Arrays.asList(commitInstant));
return paths.values().stream().flatMap(path ->
- BaseFileUtils.getInstance(path).readAvroRecords(context.getHadoopConf().get(), new Path(path)).stream())
+ BaseFileUtils.getInstance(path).readAvroRecords(context.getHadoopConf().get(), new StoragePath(path)).stream())
.filter(record -> {
if (filterByCommitTime) {
Object commitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD);
@@ -916,28 +931,35 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
for (HoodieInstant commit : commitsToReturn) {
HoodieCommitMetadata metadata =
HoodieCommitMetadata.fromBytes(commitTimeline.getInstantDetails(commit).get(), HoodieCommitMetadata.class);
- fileIdToFullPath.putAll(metadata.getFileIdAndFullPaths(new Path(basePath)));
+ fileIdToFullPath.putAll(metadata.getFileIdAndFullPaths(new StoragePath(basePath)));
}
return fileIdToFullPath;
}
- public long countRowsInPaths(String basePath, FileSystem fs, String... paths) {
+ public long countRowsInPaths(String basePath, HoodieStorage storage, String... paths) {
try {
- List<HoodieBaseFile> latestFiles = getLatestBaseFiles(basePath, fs, paths);
- return latestFiles.stream().mapToLong(baseFile -> BaseFileUtils.getInstance(baseFile.getPath()).readAvroRecords(context.getHadoopConf().get(), new Path(baseFile.getPath())).size()).sum();
+ List<HoodieBaseFile> latestFiles = getLatestBaseFiles(basePath, storage, paths);
+ return latestFiles.stream().mapToLong(baseFile ->
+ BaseFileUtils.getInstance(baseFile.getPath())
+ .readAvroRecords(context.getHadoopConf().get(), new StoragePath(baseFile.getPath())).size())
+ .sum();
} catch (Exception e) {
throw new HoodieException("Error reading hoodie table as a dataframe", e);
}
}
- public static List<HoodieBaseFile> getLatestBaseFiles(String basePath, FileSystem fs,
+ public static List<HoodieBaseFile> getLatestBaseFiles(String basePath, HoodieStorage storage,
String... paths) {
List<HoodieBaseFile> latestFiles = new ArrayList<>();
try {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(fs.getConf()).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
+ .setConf((Configuration) storage.getConf())
+ .setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
for (String path : paths) {
- TableFileSystemView.BaseFileOnlyView fileSystemView = new HoodieTableFileSystemView(metaClient,
- metaClient.getCommitsTimeline().filterCompletedInstants(), fs.globStatus(new Path(path)));
+ TableFileSystemView.BaseFileOnlyView fileSystemView =
+ new HoodieTableFileSystemView(metaClient,
+ metaClient.getCommitsTimeline().filterCompletedInstants(),
+ storage.globEntries(new StoragePath(path)));
latestFiles.addAll(fileSystemView.getLatestBaseFiles().collect(Collectors.toList()));
}
} catch (Exception e) {
@@ -958,7 +980,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
HashMap<String, String> fileIdToFullPath = getLatestFileIDsToFullPath(basePath, commitTimeline, commitsToReturn);
String[] paths = fileIdToFullPath.values().toArray(new String[fileIdToFullPath.size()]);
if (paths[0].endsWith(HoodieFileFormat.PARQUET.getFileExtension())) {
- return Arrays.stream(paths).flatMap(path -> BaseFileUtils.getInstance(path).readAvroRecords(context.getHadoopConf().get(), new Path(path)).stream())
+ return Arrays.stream(paths).flatMap(path -> BaseFileUtils.getInstance(path).readAvroRecords(context.getHadoopConf().get(), new StoragePath(path)).stream())
.filter(record -> {
if (lastCommitTimeOpt.isPresent()) {
Object commitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD);
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java
index 8de67be6bf7..c5cccd44d3d 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java
@@ -41,11 +41,11 @@ import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.metadata.JavaHoodieBackedTableMetadataWriter;
import org.apache.hudi.metrics.MetricsReporterType;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieJavaTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -95,7 +95,7 @@ public class TestHoodieMetadataBase extends HoodieJavaClientTestHarness {
this.tableType = tableType;
initPath();
initFileSystem(basePath, hadoopConf);
- fs.mkdirs(new Path(basePath));
+ storage.createDirectory(new StoragePath(basePath));
initMetaClient(tableType);
initTestDataGenerator();
metadataTableBasePath = HoodieTableMetadata.getMetadataTableBasePath(basePath);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
index d52ced8db59..a7dbd1bb4d0 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
@@ -60,13 +60,13 @@ import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.table.action.cluster.strategy.ClusteringExecutionStrategy;
import org.apache.avro.Schema;
-import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
@@ -306,7 +306,7 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
try {
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(config.getSchema()));
HoodieMergedLogRecordScanner scanner = HoodieMergedLogRecordScanner.newBuilder()
- .withFileSystem(table.getMetaClient().getFs())
+ .withStorage(table.getMetaClient().getStorage())
.withBasePath(table.getMetaClient().getBasePath())
.withLogFilePaths(clusteringOp.getDeltaFilePaths())
.withReaderSchema(readerSchema)
@@ -386,7 +386,7 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
private HoodieFileReader getBaseOrBootstrapFileReader(SerializableConfiguration hadoopConf, String bootstrapBasePath, Option<String[]> partitionFields, ClusteringOperation clusteringOp)
throws IOException {
HoodieFileReader baseFileReader = HoodieFileReaderFactory.getReaderFactory(recordType)
- .getFileReader(writeConfig, hadoopConf.get(), new Path(clusteringOp.getDataFilePath()));
+ .getFileReader(writeConfig, hadoopConf.get(), new StoragePath(clusteringOp.getDataFilePath()));
// handle bootstrap path
if (StringUtils.nonEmpty(clusteringOp.getBootstrapFilePath()) && StringUtils.nonEmpty(bootstrapBasePath)) {
String bootstrapFilePath = clusteringOp.getBootstrapFilePath();
@@ -399,7 +399,7 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
baseFileReader = HoodieFileReaderFactory.getReaderFactory(recordType).newBootstrapFileReader(
baseFileReader,
HoodieFileReaderFactory.getReaderFactory(recordType).getFileReader(
- writeConfig, hadoopConf.get(), new Path(bootstrapFilePath)), partitionFields,
+ writeConfig, hadoopConf.get(), new StoragePath(bootstrapFilePath)), partitionFields,
partitionValues);
}
return baseFileReader;
@@ -416,7 +416,7 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
boolean hasLogFiles = clusteringOps.stream().anyMatch(op -> op.getDeltaFilePaths().size() > 0);
SQLContext sqlContext = new SQLContext(jsc.sc());
- Path[] baseFilePaths = clusteringOps
+ StoragePath[] baseFilePaths = clusteringOps
.stream()
.map(op -> {
ArrayList<String> readPaths = new ArrayList<>();
@@ -429,32 +429,33 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
})
.flatMap(Collection::stream)
.filter(path -> !path.isEmpty())
- .map(Path::new)
- .toArray(Path[]::new);
+ .map(StoragePath::new)
+ .toArray(StoragePath[]::new);
HashMap<String, String> params = new HashMap<>();
params.put("hoodie.datasource.query.type", "snapshot");
params.put(TIMESTAMP_AS_OF.key(), instantTime);
- Path[] paths;
+ StoragePath[] paths;
if (hasLogFiles) {
String rawFractionConfig = getWriteConfig().getString(HoodieMemoryConfig.MAX_MEMORY_FRACTION_FOR_COMPACTION);
String compactionFractor = rawFractionConfig != null
? rawFractionConfig : HoodieMemoryConfig.DEFAULT_MR_COMPACTION_MEMORY_FRACTION;
params.put(HoodieMemoryConfig.MAX_MEMORY_FRACTION_FOR_COMPACTION.key(), compactionFractor);
- Path[] deltaPaths = clusteringOps
+ StoragePath[] deltaPaths = clusteringOps
.stream()
.filter(op -> !op.getDeltaFilePaths().isEmpty())
.flatMap(op -> op.getDeltaFilePaths().stream())
- .map(Path::new)
- .toArray(Path[]::new);
+ .map(StoragePath::new)
+ .toArray(StoragePath[]::new);
paths = CollectionUtils.combine(baseFilePaths, deltaPaths);
} else {
paths = baseFilePaths;
}
- String readPathString = String.join(",", Arrays.stream(paths).map(Path::toString).toArray(String[]::new));
+ String readPathString =
+ String.join(",", Arrays.stream(paths).map(StoragePath::toString).toArray(String[]::new));
params.put("hoodie.datasource.read.paths", readPathString);
// Building HoodieFileIndex needs this param to decide query path
params.put("glob.paths", readPathString);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
index 98c016dfaf5..fa2af5d5b90 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
@@ -42,13 +42,13 @@ import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.table.action.cluster.strategy.ClusteringExecutionStrategy;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.broadcast.Broadcast;
@@ -147,7 +147,7 @@ public abstract class SingleSparkJobExecutionStrategy<T>
Iterable<HoodieRecord<T>> indexedRecords = () -> {
try {
HoodieFileReader baseFileReader = HoodieFileReaderFactory.getReaderFactory(recordType)
- .getFileReader(writeConfig, getHoodieTable().getHadoopConf(), new Path(clusteringOp.getDataFilePath()));
+ .getFileReader(writeConfig, getHoodieTable().getHadoopConf(), new StoragePath(clusteringOp.getDataFilePath()));
Option<BaseKeyGenerator> keyGeneratorOp =
writeConfig.populateMetaFields() ? Option.empty() : Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(writeConfig.getProps()));
// NOTE: Record have to be cloned here to make sure if it holds low-level engine-specific
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java
index 3f10bde6f46..432708c6fb3 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java
@@ -155,10 +155,15 @@ public class SparkMetadataWriterUtils {
List<HoodieColumnRangeMetadata<Comparable>> columnRangeMetadataList,
long fileSize,
Path filePath) {
- Dataset<Row> fileDf = readRecordsAsRow(new Path[] {filePath}, sqlContext, metaClient, readerSchema);
+ Dataset<Row> fileDf = readRecordsAsRow(
+ new StoragePath[] {new StoragePath(filePath.toUri())},
+ sqlContext,
+ metaClient,
+ readerSchema);
Column indexedColumn = functionalIndex.apply(Arrays.asList(fileDf.col(columnToIndex)));
fileDf = fileDf.withColumn(columnToIndex, indexedColumn);
- HoodieColumnRangeMetadata<Comparable> columnRangeMetadata = computeColumnRangeMetadata(fileDf, columnToIndex, filePath.toString(), fileSize);
+ HoodieColumnRangeMetadata<Comparable> columnRangeMetadata =
+ computeColumnRangeMetadata(fileDf, columnToIndex, filePath.toString(), fileSize);
columnRangeMetadataList.add(columnRangeMetadata);
}
@@ -173,7 +178,9 @@ public class SparkMetadataWriterUtils {
HoodieWriteConfig writeConfig,
String partitionName,
String instantTime) {
- Dataset<Row> fileDf = readRecordsAsRow(new Path[] {filePath}, sqlContext, metaClient, readerSchema);
+ Dataset<Row> fileDf =
+ readRecordsAsRow(new StoragePath[] {new StoragePath(filePath.toUri())},
+ sqlContext, metaClient, readerSchema);
Column indexedColumn = functionalIndex.apply(Arrays.asList(fileDf.col(columnToIndex)));
fileDf = fileDf.withColumn(columnToIndex, indexedColumn);
BloomFilter bloomFilter = HoodieFileWriterFactory.createBloomFilter(writeConfig);
@@ -183,11 +190,14 @@ public class SparkMetadataWriterUtils {
});
ByteBuffer bloomByteBuffer = ByteBuffer.wrap(getUTF8Bytes(bloomFilter.serializeToString()));
bloomFilterMetadataList.add(createBloomFilterMetadataRecord(
- partitionName, filePath.toString(), instantTime, writeConfig.getBloomFilterType(), bloomByteBuffer, false));
+ partitionName, filePath.toString(), instantTime, writeConfig.getBloomFilterType(),
+ bloomByteBuffer, false));
}
- private static Dataset<Row> readRecordsAsRow(Path[] paths, SQLContext sqlContext, HoodieTableMetaClient metaClient, Schema schema) {
- String readPathString = String.join(",", Arrays.stream(paths).map(Path::toString).toArray(String[]::new));
+ private static Dataset<Row> readRecordsAsRow(StoragePath[] paths, SQLContext sqlContext,
+ HoodieTableMetaClient metaClient, Schema schema) {
+ String readPathString =
+ String.join(",", Arrays.stream(paths).map(StoragePath::toString).toArray(String[]::new));
HashMap<String, String> params = new HashMap<>();
params.put(QUERY_TYPE_CONFIG, QUERY_TYPE_SNAPSHOT);
params.put(READ_PATHS_CONFIG, readPathString);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
index 403eb35926c..163c2ff8b12 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
@@ -29,8 +29,8 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.index.HoodieIndexUtils;
import org.apache.hudi.io.HoodieKeyLookupResult;
+import org.apache.hudi.storage.StoragePath;
-import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.broadcast.Broadcast;
import org.slf4j.Logger;
@@ -128,7 +128,7 @@ public class HoodieFileProbingFunction implements
final HoodieBaseFile dataFile = fileIDBaseFileMap.get(fileId);
List<Pair<String, Long>> matchingKeysAndPositions = HoodieIndexUtils.filterKeysFromFile(
- new Path(dataFile.getPath()), candidateRecordKeys, hadoopConf.get());
+ new StoragePath(dataFile.getPath()), candidateRecordKeys, hadoopConf.get());
LOG.debug(
String.format("Bloom filter candidates (%d) / false positives (%d), actual matches (%d)",
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/SparkHoodieBloomIndexHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/SparkHoodieBloomIndexHelper.java
index 0b3414a8a2d..39289a02ca5 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/SparkHoodieBloomIndexHelper.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/SparkHoodieBloomIndexHelper.java
@@ -39,9 +39,9 @@ import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.HoodieKeyLookupResult;
import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.spark.Partitioner;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
@@ -51,7 +51,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
@@ -218,10 +217,11 @@ public class SparkHoodieBloomIndexHelper extends BaseHoodieBloomIndexHelper {
String.format("%s/%s", hoodieTable.getMetaClient().getBasePathV2(), partitionPath))
.collect(Collectors.toList());
- FileStatus[] allFiles =
- hoodieTable.getMetadataTable().getAllFilesInPartitions(fullPartitionPaths).values().stream()
- .flatMap(Arrays::stream)
- .toArray(FileStatus[]::new);
+ List<StoragePathInfo> allFiles =
+ hoodieTable.getMetadataTable().getAllFilesInPartitions(fullPartitionPaths).values()
+ .stream()
+ .flatMap(e -> e.stream())
+ .collect(Collectors.toList());
return new HoodieTableFileSystemView(hoodieTable.getMetaClient(), hoodieTable.getActiveTimeline(), allFiles);
} catch (IOException e) {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java
index d06b6913905..57c322e6b5d 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReaderFactory.java
@@ -21,10 +21,10 @@ package org.apache.hudi.io.storage;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.spark.sql.internal.SQLConf;
import java.io.IOException;
@@ -32,7 +32,7 @@ import java.io.IOException;
public class HoodieSparkFileReaderFactory extends HoodieFileReaderFactory {
@Override
- public HoodieFileReader newParquetFileReader(Configuration conf, Path path) {
+ public HoodieFileReader newParquetFileReader(Configuration conf, StoragePath path) {
conf.setIfUnset(SQLConf.PARQUET_BINARY_AS_STRING().key(), SQLConf.PARQUET_BINARY_AS_STRING().defaultValueString());
conf.setIfUnset(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), SQLConf.PARQUET_INT96_AS_TIMESTAMP().defaultValueString());
conf.setIfUnset(SQLConf.CASE_SENSITIVE().key(), SQLConf.CASE_SENSITIVE().defaultValueString());
@@ -47,13 +47,13 @@ public class HoodieSparkFileReaderFactory extends HoodieFileReaderFactory {
@Override
protected HoodieFileReader newHFileFileReader(HoodieConfig hoodieConfig,
Configuration conf,
- Path path,
+ StoragePath path,
Option<Schema> schemaOption) throws IOException {
throw new HoodieIOException("Not support read HFile");
}
@Override
- protected HoodieFileReader newOrcFileReader(Configuration conf, Path path) {
+ protected HoodieFileReader newOrcFileReader(Configuration conf, StoragePath path) {
throw new HoodieIOException("Not support read orc file");
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java
index 7091c2b240f..ba04e023125 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java
@@ -27,11 +27,11 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.storage.row.HoodieRowParquetConfig;
import org.apache.hudi.io.storage.row.HoodieRowParquetWriteSupport;
+import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.spark.sql.HoodieInternalRowUtils;
import org.apache.spark.sql.types.StructType;
@@ -42,7 +42,7 @@ public class HoodieSparkFileWriterFactory extends HoodieFileWriterFactory {
@Override
protected HoodieFileWriter newParquetFileWriter(
- String instantTime, Path path, Configuration conf, HoodieConfig config, Schema schema,
+ String instantTime, StoragePath path, Configuration conf, HoodieConfig config, Schema schema,
TaskContextSupplier taskContextSupplier) throws IOException {
boolean populateMetaFields = config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS);
String compressionCodecName = config.getStringOrDefault(HoodieStorageConfig.PARQUET_COMPRESSION_CODEC_NAME);
@@ -86,14 +86,14 @@ public class HoodieSparkFileWriterFactory extends HoodieFileWriterFactory {
}
@Override
- protected HoodieFileWriter newHFileFileWriter(String instantTime, Path path, Configuration conf, HoodieConfig config, Schema schema,
- TaskContextSupplier taskContextSupplier) throws IOException {
+ protected HoodieFileWriter newHFileFileWriter(String instantTime, StoragePath path, Configuration conf, HoodieConfig config, Schema schema,
+ TaskContextSupplier taskContextSupplier) throws IOException {
throw new HoodieIOException("Not support write to HFile");
}
@Override
- protected HoodieFileWriter newOrcFileWriter(String instantTime, Path path, Configuration conf, HoodieConfig config, Schema schema,
- TaskContextSupplier taskContextSupplier) throws IOException {
+ protected HoodieFileWriter newOrcFileWriter(String instantTime, StoragePath path, Configuration conf, HoodieConfig config, Schema schema,
+ TaskContextSupplier taskContextSupplier) throws IOException {
throw new HoodieIOException("Not support write to Orc file");
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java
index 79444d179ce..06d93504dba 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetReader.java
@@ -31,6 +31,7 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.common.util.collection.CloseableMappingIterator;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
@@ -57,12 +58,12 @@ import static org.apache.parquet.avro.AvroSchemaConverter.ADD_LIST_ELEMENT_RECOR
public class HoodieSparkParquetReader implements HoodieSparkFileReader {
- private final Path path;
+ private final StoragePath path;
private final Configuration conf;
private final BaseFileUtils parquetUtils;
private List<ParquetReaderIterator> readerIterators = new ArrayList<>();
- public HoodieSparkParquetReader(Configuration conf, Path path) {
+ public HoodieSparkParquetReader(Configuration conf, StoragePath path) {
this.path = path;
this.conf = new Configuration(conf);
// Avoid adding record in list element when convert parquet schema to avro schema
@@ -125,7 +126,7 @@ public class HoodieSparkParquetReader implements HoodieSparkFileReader {
conf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA(), requestedStructType.json());
conf.setBoolean(SQLConf.PARQUET_BINARY_AS_STRING().key(), (Boolean) SQLConf.get().getConf(SQLConf.PARQUET_BINARY_AS_STRING()));
conf.setBoolean(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), (Boolean) SQLConf.get().getConf(SQLConf.PARQUET_INT96_AS_TIMESTAMP()));
- ParquetReader<InternalRow> reader = ParquetReader.<InternalRow>builder((ReadSupport) new ParquetReadSupport(), path)
+ ParquetReader<InternalRow> reader = ParquetReader.<InternalRow>builder((ReadSupport) new ParquetReadSupport(), new Path(path.toUri()))
.withConf(conf)
.build();
ParquetReaderIterator<InternalRow> parquetReaderIterator = new ParquetReaderIterator<>(reader);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java
index d601e6ded3e..09f8d8dbe1c 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java
@@ -18,12 +18,13 @@
package org.apache.hudi.io.storage;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.io.storage.row.HoodieRowParquetConfig;
import org.apache.hudi.io.storage.row.HoodieRowParquetWriteSupport;
+import org.apache.hudi.storage.StoragePath;
+
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.unsafe.types.UTF8String;
@@ -47,7 +48,7 @@ public class HoodieSparkParquetWriter extends HoodieBaseParquetWriter<InternalRo
private final Function<Long, String> seqIdGenerator;
- public HoodieSparkParquetWriter(Path file,
+ public HoodieSparkParquetWriter(StoragePath file,
HoodieRowParquetConfig parquetConfig,
String instantTime,
TaskContextSupplier taskContextSupplier,
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowFileWriterFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowFileWriterFactory.java
index ad362d17014..f83780a3f09 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowFileWriterFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowFileWriterFactory.java
@@ -18,14 +18,15 @@
package org.apache.hudi.io.storage.row;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.io.storage.HoodieParquetConfig;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
+
import org.apache.spark.sql.types.StructType;
import java.io.IOException;
@@ -47,7 +48,7 @@ public class HoodieInternalRowFileWriterFactory {
* @throws IOException if format is not supported or if any exception during instantiating the RowFileWriter.
*
*/
- public static HoodieInternalRowFileWriter getInternalRowFileWriter(Path path,
+ public static HoodieInternalRowFileWriter getInternalRowFileWriter(StoragePath path,
HoodieTable hoodieTable,
HoodieWriteConfig writeConfig,
StructType schema)
@@ -59,7 +60,7 @@ public class HoodieInternalRowFileWriterFactory {
throw new UnsupportedOperationException(extension + " format not supported yet.");
}
- private static HoodieInternalRowFileWriter newParquetInternalRowFileWriter(Path path,
+ private static HoodieInternalRowFileWriter newParquetInternalRowFileWriter(StoragePath path,
HoodieTable table,
HoodieWriteConfig writeConfig,
StructType structType,
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowParquetWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowParquetWriter.java
index a7cacd055a6..dcb1f197a04 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowParquetWriter.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowParquetWriter.java
@@ -18,9 +18,10 @@
package org.apache.hudi.io.storage.row;
-import org.apache.hadoop.fs.Path;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.io.storage.HoodieBaseParquetWriter;
+import org.apache.hudi.io.storage.HoodieParquetConfig;
+import org.apache.hudi.storage.StoragePath;
+
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.unsafe.types.UTF8String;
@@ -34,7 +35,8 @@ public class HoodieInternalRowParquetWriter extends HoodieBaseParquetWriter<Inte
private final HoodieRowParquetWriteSupport writeSupport;
- public HoodieInternalRowParquetWriter(Path file, HoodieParquetConfig<HoodieRowParquetWriteSupport> parquetConfig)
+ public HoodieInternalRowParquetWriter(StoragePath file,
+ HoodieParquetConfig<HoodieRowParquetWriteSupport> parquetConfig)
throws IOException {
super(file, parquetConfig);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
index da0d3a4fe0b..98341bf62b4 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
@@ -35,6 +35,8 @@ import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieInsertException;
import org.apache.hudi.hadoop.fs.CachingPath;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.WriteMarkersFactory;
@@ -113,10 +115,12 @@ public class HoodieRowCreateHandle implements Serializable {
this.currTimer = HoodieTimer.start();
- FileSystem fs = table.getMetaClient().getFs();
+ HoodieStorage storage = table.getMetaClient().getStorage();
+ FileSystem fs = (FileSystem) storage.getFileSystem();
String writeToken = getWriteToken(taskPartitionId, taskId, taskEpochId);
- String fileName = FSUtils.makeBaseFileName(instantTime, writeToken, this.fileId, table.getBaseFileExtension());
+ String fileName = FSUtils.makeBaseFileName(instantTime, writeToken, this.fileId,
+ table.getBaseFileExtension());
this.path = makeNewPath(fs, partitionPath, fileName, writeConfig);
this.populateMetaFields = writeConfig.populateMetaFields();
@@ -134,16 +138,17 @@ public class HoodieRowCreateHandle implements Serializable {
try {
HoodiePartitionMetadata partitionMetadata =
new HoodiePartitionMetadata(
- fs,
+ storage,
instantTime,
- new Path(writeConfig.getBasePath()),
+ new StoragePath(writeConfig.getBasePath()),
FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath),
table.getPartitionMetafileFormat());
partitionMetadata.trySave(taskPartitionId);
createMarkerFile(partitionPath, fileName, instantTime, table, writeConfig);
- this.fileWriter = HoodieInternalRowFileWriterFactory.getInternalRowFileWriter(path, table, writeConfig, structType);
+ this.fileWriter = HoodieInternalRowFileWriterFactory.getInternalRowFileWriter(
+ new StoragePath(path.toUri()), table, writeConfig, structType);
} catch (IOException e) {
throw new HoodieInsertException("Failed to initialize file writer for path " + path, e);
}
@@ -237,8 +242,9 @@ public class HoodieRowCreateHandle implements Serializable {
stat.setNumInserts(writeStatus.getTotalRecords());
stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT);
stat.setFileId(fileId);
- stat.setPath(new Path(writeConfig.getBasePath()), path);
- long fileSizeInBytes = FSUtils.getFileSize(table.getMetaClient().getFs(), path);
+ stat.setPath(new StoragePath(writeConfig.getBasePath()), new StoragePath(path.toUri()));
+ long fileSizeInBytes = FSUtils.getFileSize(table.getMetaClient().getStorage(),
+ new StoragePath(path.toUri()));
stat.setTotalWriteBytes(fileSizeInBytes);
stat.setFileSizeInBytes(fileSizeInBytes);
stat.setTotalWriteErrors(writeStatus.getTotalErrorRecords());
@@ -256,7 +262,7 @@ public class HoodieRowCreateHandle implements Serializable {
}
private static Path makeNewPath(FileSystem fs, String partitionPath, String fileName, HoodieWriteConfig writeConfig) {
- Path path = FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath);
+ Path path = FSUtils.getPartitionPathInHadoopPath(writeConfig.getBasePath(), partitionPath);
try {
if (!fs.exists(path)) {
fs.mkdirs(path); // create a new partition as needed.
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
index fc3aba63740..78f1e1eb044 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
@@ -38,8 +38,9 @@ import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
-import org.apache.hadoop.fs.Path;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.action.commit.HoodieMergeHelper;
+
import org.apache.spark.TaskContext;
import org.apache.spark.TaskContext$;
@@ -108,7 +109,7 @@ public abstract class HoodieSparkTable<T>
context.getHadoopConf().get(), config, failedWritesCleaningPolicy, context,
Option.of(triggeringInstantTimestamp));
try {
- if (isMetadataTableExists || metaClient.getFs().exists(new Path(
+ if (isMetadataTableExists || metaClient.getStorage().exists(new StoragePath(
HoodieTableMetadata.getMetadataTableBasePath(metaClient.getBasePath())))) {
isMetadataTableExists = true;
return Option.of(metadataWriter);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java
index ffda89d5b7f..a36111c8341 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java
@@ -20,8 +20,8 @@ package org.apache.hudi.table.action.bootstrap;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.avro.model.HoodieFileStatus;
+import org.apache.hudi.avro.model.HoodiePath;
import org.apache.hudi.client.bootstrap.BootstrapWriteStatus;
-import org.apache.hudi.common.bootstrap.FileStatusUtils;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.BootstrapFileMapping;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
@@ -29,10 +29,10 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.io.HoodieBootstrapHandle;
import org.apache.hudi.keygen.KeyGeneratorInterface;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,7 +53,8 @@ public abstract class BaseBootstrapMetadataHandler implements BootstrapMetadataH
}
public BootstrapWriteStatus runMetadataBootstrap(String srcPartitionPath, String partitionPath, KeyGeneratorInterface keyGenerator) {
- Path sourceFilePath = FileStatusUtils.toPath(srcFileStatus.getPath());
+ HoodiePath path = srcFileStatus.getPath();
+ StoragePath sourceFilePath = path != null ? new StoragePath(path.getUri()) : null;
HoodieBootstrapHandle<?, ?, ?, ?> bootstrapHandle = new HoodieBootstrapHandle(config, HoodieTimeline.METADATA_BOOTSTRAP_INSTANT_TS,
table, partitionPath, FSUtils.createNewFileIdPfx(), table.getTaskContextSupplier());
try {
@@ -78,8 +79,8 @@ public abstract class BaseBootstrapMetadataHandler implements BootstrapMetadataH
return writeStatus;
}
- abstract Schema getAvroSchema(Path sourceFilePath) throws IOException;
+ abstract Schema getAvroSchema(StoragePath sourceFilePath) throws IOException;
abstract void executeBootstrap(HoodieBootstrapHandle<?, ?, ?, ?> bootstrapHandle,
- Path sourceFilePath, KeyGeneratorInterface keyGenerator, String partitionPath, Schema avroSchema) throws Exception;
+ StoragePath sourceFilePath, KeyGeneratorInterface keyGenerator, String partitionPath, Schema avroSchema) throws Exception;
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/OrcBootstrapMetadataHandler.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/OrcBootstrapMetadataHandler.java
index fa60148ea10..6e40eef6522 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/OrcBootstrapMetadataHandler.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/OrcBootstrapMetadataHandler.java
@@ -31,6 +31,7 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.io.HoodieBootstrapHandle;
import org.apache.hudi.keygen.KeyGeneratorInterface;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.util.ExecutorFactory;
@@ -57,20 +58,23 @@ class OrcBootstrapMetadataHandler extends BaseBootstrapMetadataHandler {
}
@Override
- Schema getAvroSchema(Path sourceFilePath) throws IOException {
- Reader orcReader = OrcFile.createReader(sourceFilePath, OrcFile.readerOptions(table.getHadoopConf()));
+ Schema getAvroSchema(StoragePath sourceFilePath) throws IOException {
+ Reader orcReader = OrcFile.createReader(
+ new Path(sourceFilePath.toUri()), OrcFile.readerOptions(table.getHadoopConf()));
TypeDescription orcSchema = orcReader.getSchema();
return AvroOrcUtils.createAvroSchema(orcSchema);
}
@Override
- void executeBootstrap(HoodieBootstrapHandle<?, ?, ?, ?> bootstrapHandle, Path sourceFilePath, KeyGeneratorInterface keyGenerator,
+ void executeBootstrap(HoodieBootstrapHandle<?, ?, ?, ?> bootstrapHandle,
+ StoragePath sourceFilePath, KeyGeneratorInterface keyGenerator,
String partitionPath, Schema avroSchema) throws Exception {
// TODO support spark orc reader
if (config.getRecordMerger().getRecordType() == HoodieRecordType.SPARK) {
throw new UnsupportedOperationException();
}
- Reader orcReader = OrcFile.createReader(sourceFilePath, OrcFile.readerOptions(table.getHadoopConf()));
+ Reader orcReader = OrcFile.createReader(
+ new Path(sourceFilePath.toUri()), OrcFile.readerOptions(table.getHadoopConf()));
TypeDescription orcSchema = AvroOrcUtils.createOrcSchema(avroSchema);
HoodieExecutor<Void> executor = null;
RecordReader reader = orcReader.rows(new Reader.Options(table.getHadoopConf()).schema(orcSchema));
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java
index 80a7e6a86a7..3aad5ecd821 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/ParquetBootstrapMetadataHandler.java
@@ -33,6 +33,7 @@ import org.apache.hudi.io.HoodieBootstrapHandle;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.keygen.KeyGeneratorInterface;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.util.ExecutorFactory;
@@ -64,8 +65,9 @@ class ParquetBootstrapMetadataHandler extends BaseBootstrapMetadataHandler {
}
@Override
- Schema getAvroSchema(Path sourceFilePath) throws IOException {
- ParquetMetadata readFooter = ParquetFileReader.readFooter(table.getHadoopConf(), sourceFilePath,
+ Schema getAvroSchema(StoragePath sourceFilePath) throws IOException {
+ ParquetMetadata readFooter = ParquetFileReader.readFooter(
+ table.getHadoopConf(), new Path(sourceFilePath.toUri()),
ParquetMetadataConverter.NO_FILTER);
MessageType parquetSchema = readFooter.getFileMetaData().getSchema();
return new AvroSchemaConverter().convert(parquetSchema);
@@ -73,14 +75,14 @@ class ParquetBootstrapMetadataHandler extends BaseBootstrapMetadataHandler {
@Override
protected void executeBootstrap(HoodieBootstrapHandle<?, ?, ?, ?> bootstrapHandle,
- Path sourceFilePath,
+ StoragePath sourceFilePath,
KeyGeneratorInterface keyGenerator,
String partitionPath,
Schema schema) throws Exception {
HoodieRecord.HoodieRecordType recordType = table.getConfig().getRecordMerger().getRecordType();
HoodieFileReader reader = HoodieFileReaderFactory.getReaderFactory(recordType)
- .getFileReader(table.getConfig(), table.getHadoopConf(), sourceFilePath);
+ .getFileReader(table.getConfig(), table.getHadoopConf(), sourceFilePath);
HoodieExecutor<Void> executor = null;
try {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
index b45a691fbad..c51bb5f21c4 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
@@ -31,13 +31,12 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.data.HoodieJavaPairRDD;
import org.apache.hudi.exception.HoodieDeletePartitionException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.WorkloadProfile;
import org.apache.hudi.table.WorkloadStat;
import org.apache.hudi.table.action.HoodieWriteMetadata;
-import org.apache.hadoop.fs.Path;
-
import java.time.Duration;
import java.util.Collections;
import java.util.HashMap;
@@ -67,25 +66,31 @@ public class SparkDeletePartitionCommitActionExecutor<T>
context.setJobStatus(this.getClass().getSimpleName(), "Gather all file ids from all deleting partitions.");
Map<String, List<String>> partitionToReplaceFileIds =
HoodieJavaPairRDD.getJavaPairRDD(context.parallelize(partitions).distinct()
- .mapToPair(partitionPath -> Pair.of(partitionPath, getAllExistingFileIds(partitionPath)))).collectAsMap();
+ .mapToPair(
+ partitionPath -> Pair.of(partitionPath, getAllExistingFileIds(partitionPath))))
+ .collectAsMap();
HoodieWriteMetadata<HoodieData<WriteStatus>> result = new HoodieWriteMetadata<>();
result.setPartitionToReplaceFileIds(partitionToReplaceFileIds);
result.setIndexUpdateDuration(Duration.ofMillis(timer.endTimer()));
result.setWriteStatuses(context.emptyHoodieData());
// created requested
- HoodieInstant dropPartitionsInstant = new HoodieInstant(REQUESTED, REPLACE_COMMIT_ACTION, instantTime);
- if (!table.getMetaClient().getFs().exists(new Path(table.getMetaClient().getMetaPath(),
- dropPartitionsInstant.getFileName()))) {
- HoodieRequestedReplaceMetadata requestedReplaceMetadata = HoodieRequestedReplaceMetadata.newBuilder()
- .setOperationType(WriteOperationType.DELETE_PARTITION.name())
- .setExtraMetadata(extraMetadata.orElse(Collections.emptyMap()))
- .build();
+ HoodieInstant dropPartitionsInstant =
+ new HoodieInstant(REQUESTED, REPLACE_COMMIT_ACTION, instantTime);
+ if (!table.getMetaClient().getStorage().exists(
+ new StoragePath(table.getMetaClient().getMetaPath(),
+ dropPartitionsInstant.getFileName()))) {
+ HoodieRequestedReplaceMetadata requestedReplaceMetadata =
+ HoodieRequestedReplaceMetadata.newBuilder()
+ .setOperationType(WriteOperationType.DELETE_PARTITION.name())
+ .setExtraMetadata(extraMetadata.orElse(Collections.emptyMap()))
+ .build();
table.getMetaClient().getActiveTimeline().saveToPendingReplaceCommit(dropPartitionsInstant,
TimelineMetadataUtils.serializeRequestedReplaceMetadata(requestedReplaceMetadata));
}
- this.saveWorkloadProfileMetadataToInflight(new WorkloadProfile(Pair.of(new HashMap<>(), new WorkloadStat())),
+ this.saveWorkloadProfileMetadataToInflight(
+ new WorkloadProfile(Pair.of(new HashMap<>(), new WorkloadStat())),
instantTime);
this.commitOnAutoCommit(result);
return result;
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/BaseSparkInternalRowReaderContext.java b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/BaseSparkInternalRowReaderContext.java
index d0b27e507fe..0ab47de9571 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/BaseSparkInternalRowReaderContext.java
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/BaseSparkInternalRowReaderContext.java
@@ -29,11 +29,11 @@ import org.apache.hudi.common.model.HoodieSparkRecord;
import org.apache.hudi.common.util.ConfigUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.spark.sql.HoodieInternalRowUtils;
import org.apache.spark.sql.HoodieUnsafeRowUtils;
import org.apache.spark.sql.catalyst.InternalRow;
@@ -53,8 +53,8 @@ import static org.apache.spark.sql.HoodieInternalRowUtils.getCachedSchema;
*/
public abstract class BaseSparkInternalRowReaderContext extends HoodieReaderContext<InternalRow> {
@Override
- public FileSystem getFs(String path, Configuration conf) {
- return HadoopFSUtils.getFs(path, conf);
+ public HoodieStorage getStorage(String path, Configuration conf) {
+ return HoodieStorageUtils.getStorage(path, conf);
}
@Override
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala
index 963035caf21..e555f0a4835 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala
@@ -19,16 +19,18 @@
package org.apache.hudi
-import org.apache.avro.Schema
-import org.apache.avro.generic.IndexedRecord
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
import org.apache.hudi.common.engine.HoodieReaderContext
import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.util.ValidationUtils.checkState
import org.apache.hudi.common.util.collection.{ClosableIterator, CloseableMappingIterator}
import org.apache.hudi.io.storage.{HoodieSparkFileReaderFactory, HoodieSparkParquetReader}
+import org.apache.hudi.storage.StoragePath
import org.apache.hudi.util.CloseableInternalRowIterator
+
+import org.apache.avro.Schema
+import org.apache.avro.generic.IndexedRecord
+import org.apache.hadoop.conf.Configuration
+import org.apache.spark.sql.HoodieInternalRowUtils
import org.apache.spark.sql.avro.HoodieAvroDeserializer
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{JoinedRow, UnsafeProjection, UnsafeRow}
@@ -36,7 +38,6 @@ import org.apache.spark.sql.execution.datasources.PartitionedFile
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.vectorized.{ColumnVector, ColumnarBatch}
-import org.apache.spark.sql.HoodieInternalRowUtils
import scala.collection.mutable
@@ -55,7 +56,7 @@ class SparkFileFormatInternalRowReaderContext(readerMaps: mutable.Map[Long, Part
lazy val sparkFileReaderFactory = new HoodieSparkFileReaderFactory
val deserializerMap: mutable.Map[Schema, HoodieAvroDeserializer] = mutable.Map()
- override def getFileRecordIterator(filePath: Path,
+ override def getFileRecordIterator(filePath: StoragePath,
start: Long,
length: Long,
dataSchema: Schema,
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/PathUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/PathUtils.scala
index 4a96b542d58..000b256015d 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/PathUtils.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/PathUtils.scala
@@ -17,8 +17,10 @@
package org.apache.hudi.util
-import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.storage.{HoodieStorage, StoragePath}
+
+import scala.jdk.CollectionConverters.asScalaBufferConverter
/**
* TODO convert to Java, move to hudi-common
@@ -29,7 +31,7 @@ object PathUtils {
* This method copied from [[org.apache.spark.deploy.SparkHadoopUtil]].
* [[org.apache.spark.deploy.SparkHadoopUtil]] becomes private since Spark 3.0.0 and hence we had to copy it locally.
*/
- def isGlobPath(pattern: Path): Boolean = {
+ def isGlobPath(pattern: StoragePath): Boolean = {
pattern.toString.exists("{}[]*?\\".toSet.contains)
}
@@ -37,15 +39,15 @@ object PathUtils {
* This method is inspired from [[org.apache.spark.deploy.SparkHadoopUtil]] with some modifications like
* skipping meta paths.
*/
- def globPath(fs: FileSystem, pattern: Path): Seq[Path] = {
+ def globPath(storage: HoodieStorage, pattern: StoragePath): Seq[StoragePath] = {
// find base path to assist in skipping meta paths
var basePath = pattern.getParent
while (basePath.getName.equals("*")) {
basePath = basePath.getParent
}
- Option(fs.globStatus(pattern)).map { statuses => {
- val nonMetaStatuses = statuses.filterNot(entry => {
+ Option(storage.globEntries(pattern)).map { pathInfoList => {
+ val nonMetaStatuses = pathInfoList.asScala.filterNot(entry => {
// skip all entries in meta path
var leafPath = entry.getPath
// walk through every parent until we reach base path. if .hoodie is found anywhere, path needs to be skipped
@@ -54,17 +56,17 @@ object PathUtils {
}
leafPath.getName.equals(HoodieTableMetaClient.METAFOLDER_NAME)
})
- nonMetaStatuses.map(_.getPath.makeQualified(fs.getUri, fs.getWorkingDirectory)).toSeq
+ nonMetaStatuses.map(e => e.getPath.makeQualified(storage.getUri))
}
- }.getOrElse(Seq.empty[Path])
+ }.getOrElse(Seq.empty[StoragePath])
}
/**
* This method copied from [[org.apache.spark.deploy.SparkHadoopUtil]].
* [[org.apache.spark.deploy.SparkHadoopUtil]] becomes private since Spark 3.0.0 and hence we had to copy it locally.
*/
- def globPathIfNecessary(fs: FileSystem, pattern: Path): Seq[Path] = {
- if (isGlobPath(pattern)) globPath(fs, pattern) else Seq(pattern)
+ def globPathIfNecessary(storage: HoodieStorage, pattern: StoragePath): Seq[StoragePath] = {
+ if (isGlobPath(pattern)) globPath(storage, pattern) else Seq(pattern)
}
/**
@@ -72,13 +74,13 @@ object PathUtils {
* which match the glob pattern. Otherwise, returns original path
*
* @param paths List of absolute or globbed paths
- * @param fs File system
+ * @param fs {@link HoodieStorage} instance
* @return list of absolute file paths
*/
- def checkAndGlobPathIfNecessary(paths: Seq[String], fs: FileSystem): Seq[Path] = {
+ def checkAndGlobPathIfNecessary(paths: Seq[String], storage: HoodieStorage): Seq[StoragePath] = {
paths.flatMap(path => {
- val qualified = new Path(path).makeQualified(fs.getUri, fs.getWorkingDirectory)
- globPathIfNecessary(fs, qualified)
+ val qualified = new StoragePath(path).makeQualified(storage.getUri);
+ globPathIfNecessary(storage, qualified)
})
}
}
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSparkPartitionedFileUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSparkPartitionedFileUtils.scala
index 53d95f09394..57b70b0317f 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSparkPartitionedFileUtils.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSparkPartitionedFileUtils.scala
@@ -19,6 +19,8 @@
package org.apache.spark.sql.execution.datasources
+import org.apache.hudi.storage.StoragePath
+
import org.apache.hadoop.fs.{FileStatus, Path}
import org.apache.spark.sql.catalyst.InternalRow
@@ -51,7 +53,7 @@ trait HoodieSparkPartitionedFileUtils extends Serializable {
* @param partitionedFile Spark [[PartitionedFile]] instance.
* @return Hadoop [[Path]] instance.
*/
- def getPathFromPartitionedFile(partitionedFile: PartitionedFile): Path
+ def getPathFromPartitionedFile(partitionedFile: PartitionedFile): StoragePath
/**
* Gets the [[String]] path from Spark [[PartitionedFile]] instance.
@@ -71,7 +73,7 @@ trait HoodieSparkPartitionedFileUtils extends Serializable {
* @return a new [[PartitionedFile]] instance.
*/
def createPartitionedFile(partitionValues: InternalRow,
- filePath: Path,
+ filePath: StoragePath,
start: Long,
length: Long): PartitionedFile
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala
index 91fe6dabc2e..a65e4524b4e 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala
@@ -18,11 +18,12 @@
package org.apache.spark.sql.hudi
-import org.apache.avro.Schema
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
import org.apache.hudi.client.utils.SparkRowSerDe
import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.storage.StoragePath
+
+import org.apache.avro.Schema
+import org.apache.hadoop.conf.Configuration
import org.apache.spark.sql._
import org.apache.spark.sql.avro.{HoodieAvroDeserializer, HoodieAvroSchemaConverters, HoodieAvroSerializer}
import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
@@ -188,7 +189,7 @@ trait SparkAdapter extends Serializable {
def createRelation(sqlContext: SQLContext,
metaClient: HoodieTableMetaClient,
schema: Schema,
- globPaths: Array[Path],
+ globPaths: Array[StoragePath],
parameters: java.util.Map[String, String]): BaseRelation
/**
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
index 2e42f729a05..6712ce75b89 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
@@ -96,7 +96,8 @@ public class TestClientRollback extends HoodieClientTestBase {
HoodieWriteConfig cfg = getConfigBuilder().withCleanConfig(HoodieCleanConfig.newBuilder()
.withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS).retainCommits(1).build()).build();
try (SparkRDDWriteClient client = getHoodieWriteClient(cfg)) {
- HoodieTestDataGenerator.writePartitionMetadataDeprecated(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, basePath);
+ HoodieTestDataGenerator.writePartitionMetadataDeprecated(storage,
+ HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, basePath);
/**
* Write 1 (only inserts)
@@ -232,7 +233,8 @@ public class TestClientRollback extends HoodieClientTestBase {
.withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS).retainCommits(1).build())
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()).build();
try (SparkRDDWriteClient client = getHoodieWriteClient(cfg)) {
- HoodieTestDataGenerator.writePartitionMetadataDeprecated(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, basePath);
+ HoodieTestDataGenerator.writePartitionMetadataDeprecated(storage,
+ HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, basePath);
/**
* Write 1 (only inserts)
@@ -276,7 +278,8 @@ public class TestClientRollback extends HoodieClientTestBase {
HoodieWriteConfig cfg = getConfigBuilder().withCleanConfig(HoodieCleanConfig.newBuilder()
.withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS).retainFileVersions(2).build()).build();
try (SparkRDDWriteClient client = getHoodieWriteClient(cfg)) {
- HoodieTestDataGenerator.writePartitionMetadataDeprecated(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, basePath);
+ HoodieTestDataGenerator.writePartitionMetadataDeprecated(storage,
+ HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, basePath);
/**
* Write 1 (only inserts)
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
index 5abccdbf850..b57cc30b502 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
@@ -55,7 +55,6 @@ import org.apache.hudi.testutils.HoodieClientTestBase;
import org.apache.hudi.timeline.service.handlers.marker.AsyncTimelineServerBasedDetectionStrategy;
import org.apache.curator.test.TestingServer;
-import org.apache.hadoop.fs.Path;
import org.apache.spark.SparkException;
import org.apache.spark.api.java.JavaRDD;
import org.junit.jupiter.api.AfterEach;
@@ -135,9 +134,10 @@ public class TestHoodieClientMultiWriter extends HoodieClientTestBase {
initPath();
initSparkContexts();
initTestDataGenerator();
- initFileSystem();
- fs.mkdirs(new Path(basePath));
- metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ, HoodieFileFormat.PARQUET);
+ initHoodieStorage();
+ storage.createDirectory(new StoragePath(basePath));
+ metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ,
+ HoodieFileFormat.PARQUET);
initTestDataGenerator();
}
@@ -244,7 +244,8 @@ public class TestHoodieClientMultiWriter extends HoodieClientTestBase {
// this commit 003 will fail quickly because early conflict detection before create marker.
final String nextCommitTime3 = "003";
assertThrows(SparkException.class, () -> {
- final JavaRDD<WriteStatus> writeStatusList3 = startCommitForUpdate(writeConfig, client3, nextCommitTime3, 100);
+ final JavaRDD<WriteStatus> writeStatusList3 =
+ startCommitForUpdate(writeConfig, client3, nextCommitTime3, 100);
client3.commit(nextCommitTime3, writeStatusList3);
}, "Early conflict detected but cannot resolve conflicts for overlapping writes");
@@ -253,11 +254,14 @@ public class TestHoodieClientMultiWriter extends HoodieClientTestBase {
client2.commit(nextCommitTime2, writeStatusList2);
});
- HoodieWriteConfig config4 = HoodieWriteConfig.newBuilder().withProperties(writeConfig.getProps()).withHeartbeatIntervalInMs(heartBeatIntervalForCommit4).build();
+ HoodieWriteConfig config4 =
+ HoodieWriteConfig.newBuilder().withProperties(writeConfig.getProps())
+ .withHeartbeatIntervalInMs(heartBeatIntervalForCommit4).build();
final SparkRDDWriteClient client4 = getHoodieWriteClient(config4);
- Path heartbeatFilePath = new Path(HoodieTableMetaClient.getHeartbeatFolderPath(basePath) + StoragePath.SEPARATOR + nextCommitTime3);
- fs.create(heartbeatFilePath, true);
+ StoragePath heartbeatFilePath = new StoragePath(
+ HoodieTableMetaClient.getHeartbeatFolderPath(basePath) + StoragePath.SEPARATOR + nextCommitTime3);
+ storage.create(heartbeatFilePath, true);
// Wait for heart beat expired for failed commitTime3 "003"
// Otherwise commit4 still can see conflict between failed write 003.
@@ -265,7 +269,8 @@ public class TestHoodieClientMultiWriter extends HoodieClientTestBase {
final String nextCommitTime4 = "004";
assertDoesNotThrow(() -> {
- final JavaRDD<WriteStatus> writeStatusList4 = startCommitForUpdate(writeConfig, client4, nextCommitTime4, 100);
+ final JavaRDD<WriteStatus> writeStatusList4 =
+ startCommitForUpdate(writeConfig, client4, nextCommitTime4, 100);
client4.commit(nextCommitTime4, writeStatusList4);
});
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestMultiWriterWithPreferWriterIngestion.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestMultiWriterWithPreferWriterIngestion.java
index 80cc4be40d1..7f5d7780888 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestMultiWriterWithPreferWriterIngestion.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestMultiWriterWithPreferWriterIngestion.java
@@ -37,10 +37,10 @@ import org.apache.hudi.config.HoodieLockConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieClusteringException;
import org.apache.hudi.exception.HoodieWriteConflictException;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.testutils.HoodieClientTestBase;
-import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaRDD;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
@@ -68,9 +68,10 @@ public class TestMultiWriterWithPreferWriterIngestion extends HoodieClientTestBa
initPath();
initSparkContexts();
initTestDataGenerator();
- initFileSystem();
- fs.mkdirs(new Path(basePath));
- metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ, HoodieFileFormat.PARQUET);
+ initHoodieStorage();
+ storage.createDirectory(new StoragePath(basePath));
+ metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ,
+ HoodieFileFormat.PARQUET);
initTestDataGenerator();
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSimpleTransactionDirectMarkerBasedDetectionStrategyWithZKLockProvider.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSimpleTransactionDirectMarkerBasedDetectionStrategyWithZKLockProvider.java
index 62a55a3a046..96e4aac5161 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSimpleTransactionDirectMarkerBasedDetectionStrategyWithZKLockProvider.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSimpleTransactionDirectMarkerBasedDetectionStrategyWithZKLockProvider.java
@@ -72,7 +72,7 @@ public class TestSimpleTransactionDirectMarkerBasedDetectionStrategyWithZKLockPr
} else {
initTestDataGenerator(new String[] {""});
}
- initFileSystem();
+ initHoodieStorage();
metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ);
Properties properties = getPropertiesForKeyGen();
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java
index 25e9a9c2e8b..d0b35230256 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java
@@ -24,9 +24,9 @@ import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
-import org.apache.hudi.common.testutils.InProcessTimeGenerator;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.testutils.InProcessTimeGenerator;
import org.apache.hudi.common.testutils.RawTripTestPayload;
import org.apache.hudi.common.util.BaseFileUtils;
import org.apache.hudi.common.util.Option;
@@ -36,6 +36,7 @@ import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.CreateHandleFactory;
import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.io.HoodieWriteHandle;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
@@ -69,7 +70,7 @@ public class TestUpdateSchemaEvolution extends HoodieSparkClientTestHarness impl
initPath();
HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath);
initSparkContexts("TestUpdateSchemaEvolution");
- initFileSystem();
+ initHoodieStorage();
initTimelineService();
}
@@ -135,7 +136,7 @@ public class TestUpdateSchemaEvolution extends HoodieSparkClientTestHarness impl
updateRecords.iterator(), updateRecords.get(0).getPartitionPath(), insertResult.getFileId(), supplier, Option.empty());
List<GenericRecord> oldRecords = BaseFileUtils.getInstance(updateTable.getBaseFileFormat())
.readAvroRecords(updateTable.getHadoopConf(),
- new Path(updateTable.getConfig().getBasePath() + "/" + insertResult.getStat().getPath()),
+ new StoragePath(updateTable.getConfig().getBasePath() + "/" + insertResult.getStat().getPath()),
mergeHandle.getWriterSchemaWithMetaFields());
for (GenericRecord rec : oldRecords) {
// TODO create hoodie record with rec can getRecordKey
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkConsistentBucketClusteringPlanStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkConsistentBucketClusteringPlanStrategy.java
index 02708edfaad..a96bdf8cc59 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkConsistentBucketClusteringPlanStrategy.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkConsistentBucketClusteringPlanStrategy.java
@@ -19,6 +19,7 @@
package org.apache.hudi.client.clustering.plan.strategy;
import org.apache.hudi.avro.model.HoodieClusteringGroup;
+import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.ConsistentHashingNode;
import org.apache.hudi.common.model.FileSlice;
@@ -29,7 +30,6 @@ import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.collection.Triple;
import org.apache.hudi.config.HoodieIndexConfig;
-import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.index.bucket.ConsistentBucketIdentifier;
@@ -60,7 +60,7 @@ public class TestSparkConsistentBucketClusteringPlanStrategy extends HoodieSpark
private void setup() throws IOException {
initPath();
initSparkContexts();
- initFileSystem();
+ initHoodieStorage();
metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ);
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestConsistentBucketIndex.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestConsistentBucketIndex.java
index ea73a95f1ce..866dfcb5d2d 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestConsistentBucketIndex.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestConsistentBucketIndex.java
@@ -43,8 +43,8 @@ import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.table.action.compact.CompactionTriggerStrategy;
-import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
import org.apache.hudi.testutils.HoodieMergeOnReadTestUtils;
+import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
import org.apache.hudi.testutils.MetadataMergeWriteStatus;
import org.apache.hadoop.fs.FileStatus;
@@ -100,7 +100,7 @@ public class TestConsistentBucketIndex extends HoodieSparkClientTestHarness {
} else {
initTestDataGenerator(new String[] {""});
}
- initFileSystem();
+ initHoodieStorage();
Properties props = getPropertiesForKeyGen(populateMetaFields);
props.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
metaClient = HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ, props);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
index 951f4c9277d..fda9366a876 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
@@ -104,7 +104,9 @@ import org.apache.hudi.metadata.HoodieTableMetadataUtil;
import org.apache.hudi.metadata.MetadataPartitionType;
import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
import org.apache.hudi.metrics.Metrics;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.HoodieWriteMetadata;
@@ -117,9 +119,6 @@ import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.schema.MessageType;
import org.apache.spark.api.java.JavaRDD;
@@ -405,7 +404,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertTrue(metadataWriter.isPresent());
HoodieTableConfig hoodieTableConfig =
- new HoodieTableConfig(this.fs, metaClient.getMetaPath(), writeConfig.getPayloadClass(), writeConfig.getStringOrDefault(HoodieWriteConfig.RECORD_MERGER_IMPLS));
+ new HoodieTableConfig(this.storage, metaClient.getMetaPath(), writeConfig.getPayloadClass(),
+ writeConfig.getStringOrDefault(HoodieWriteConfig.RECORD_MERGER_IMPLS));
assertFalse(hoodieTableConfig.getMetadataPartitions().isEmpty());
// Turn off metadata table
@@ -415,18 +415,21 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
.build();
testTable = HoodieTestTable.of(metaClient);
String instant2 = "0000002";
- HoodieCommitMetadata hoodieCommitMetadata2 = doWriteOperationWithMeta(testTable, instant2, INSERT);
+ HoodieCommitMetadata hoodieCommitMetadata2 =
+ doWriteOperationWithMeta(testTable, instant2, INSERT);
metaClient.reloadActiveTimeline();
HoodieTable table2 = HoodieSparkTable.create(writeConfig2, context, metaClient);
Option metadataWriter2 = table2.getMetadataWriter(instant2);
assertFalse(metadataWriter2.isPresent());
HoodieTableConfig hoodieTableConfig2 =
- new HoodieTableConfig(this.fs, metaClient.getMetaPath(), writeConfig2.getPayloadClass(), writeConfig.getStringOrDefault(HoodieWriteConfig.RECORD_MERGER_IMPLS));
+ new HoodieTableConfig(this.storage, metaClient.getMetaPath(),
+ writeConfig2.getPayloadClass(),
+ writeConfig.getStringOrDefault(HoodieWriteConfig.RECORD_MERGER_IMPLS));
assertEquals(Collections.emptySet(), hoodieTableConfig2.getMetadataPartitions());
// Assert metadata table folder is deleted
- assertFalse(metaClient.getFs().exists(
- new Path(getMetadataTableBasePath(writeConfig2.getBasePath()))));
+ assertFalse(metaClient.getStorage().exists(
+ new StoragePath(getMetadataTableBasePath(writeConfig2.getBasePath()))));
// Enable metadata table again and initialize metadata table through
// HoodieTable.getMetadataWriter() function
@@ -444,7 +447,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
validateMetadata(testTable, true);
assertTrue(metadataWriter3.isPresent());
HoodieTableConfig hoodieTableConfig3 =
- new HoodieTableConfig(this.fs, metaClient.getMetaPath(), writeConfig.getPayloadClass(), writeConfig.getStringOrDefault(HoodieWriteConfig.RECORD_MERGER_IMPLS));
+ new HoodieTableConfig(this.storage, metaClient.getMetaPath(), writeConfig.getPayloadClass(),
+ writeConfig.getStringOrDefault(HoodieWriteConfig.RECORD_MERGER_IMPLS));
assertFalse(hoodieTableConfig3.getMetadataPartitions().isEmpty());
}
@@ -489,14 +493,19 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertFalse(partitions.contains(filteredDirectoryThree),
"Must not contain the filtered directory " + filteredDirectoryThree);
- FileStatus[] statuses = metadata(writeConfig, context).getAllFilesInPartition(new Path(basePath, "p1"));
- assertEquals(tableType == COPY_ON_WRITE ? 3 : 4, statuses.length);
- statuses = metadata(writeConfig, context).getAllFilesInPartition(new Path(basePath, "p2"));
- assertEquals(tableType == COPY_ON_WRITE ? 6 : 7, statuses.length);
- Map<String, FileStatus[]> partitionsToFilesMap = metadata(writeConfig, context).getAllFilesInPartitions(asList(basePath + "/p1", basePath + "/p2"));
+ List<StoragePathInfo> allFilesList = metadata(writeConfig, context)
+ .getAllFilesInPartition(new StoragePath(basePath, "p1"));
+ assertEquals(tableType == COPY_ON_WRITE ? 3 : 4, allFilesList.size());
+ allFilesList = metadata(writeConfig, context)
+ .getAllFilesInPartition(new StoragePath(basePath, "p2"));
+ assertEquals(tableType == COPY_ON_WRITE ? 6 : 7, allFilesList.size());
+ Map<String, List<StoragePathInfo>> partitionsToFilesMap = metadata(writeConfig, context)
+ .getAllFilesInPartitions(asList(basePath + "/p1", basePath + "/p2"));
assertEquals(2, partitionsToFilesMap.size());
- assertEquals(tableType == COPY_ON_WRITE ? 3 : 4, partitionsToFilesMap.get(basePath + "/p1").length);
- assertEquals(tableType == COPY_ON_WRITE ? 6 : 7, partitionsToFilesMap.get(basePath + "/p2").length);
+ assertEquals(tableType == COPY_ON_WRITE ? 3 : 4,
+ partitionsToFilesMap.get(basePath + "/p1").size());
+ assertEquals(tableType == COPY_ON_WRITE ? 6 : 7,
+ partitionsToFilesMap.get(basePath + "/p2").size());
}
/**
@@ -814,7 +823,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
HoodieBaseFile baseFile = fileSlices.get(0).getBaseFile().get();
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
HoodieFileReaderFactory.getReaderFactory(HoodieRecordType.AVRO).getFileReader(
- table.getConfig(), context.getHadoopConf().get(), new Path(baseFile.getPath()));
+ table.getConfig(), context.getHadoopConf().get(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
records.forEach(entry -> {
if (populateMetaFields) {
@@ -938,10 +947,13 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// validateMetadata(testTable);
// Fetch compaction Commit file and rename to some other file. completed compaction meta file should have some serialized info that table interprets
// for future upserts. so, renaming the file here to some temp name and later renaming it back to same name.
- java.nio.file.Path metaFilePath = Paths.get(HoodieTestUtils.getCompleteInstantPath(metaClient.getFs(),
- new Path(metadataTableBasePath, METAFOLDER_NAME), metadataCompactionInstant.get(), HoodieTimeline.COMMIT_ACTION)
+ java.nio.file.Path metaFilePath = Paths.get(HoodieTestUtils.getCompleteInstantPath(
+ metaClient.getStorage(),
+ new StoragePath(metadataTableBasePath, METAFOLDER_NAME),
+ metadataCompactionInstant.get(), HoodieTimeline.COMMIT_ACTION)
.toUri());
- java.nio.file.Path tempFilePath = FileCreateUtils.renameFileToTemp(metaFilePath, metadataCompactionInstant.get());
+ java.nio.file.Path tempFilePath =
+ FileCreateUtils.renameFileToTemp(metaFilePath, metadataCompactionInstant.get());
metaClient.reloadActiveTimeline();
testTable = HoodieMetadataTestTable.of(metaClient, metadataWriter, Option.of(context));
// this validation will exercise the code path where a compaction is inflight in metadata table, but still metadata based file listing should match non
@@ -971,8 +983,10 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// Fetch compaction Commit file and rename to some other file. completed compaction meta file should have some serialized info that table interprets
// for future upserts. so, renaming the file here to some temp name and later renaming it back to same name.
- metaFilePath = Paths.get(HoodieTestUtils.getCompleteInstantPath(metaClient.getFs(),
- new Path(metadataTableBasePath, METAFOLDER_NAME), metadataCompactionInstant.get(), HoodieTimeline.COMMIT_ACTION)
+ metaFilePath = Paths.get(HoodieTestUtils.getCompleteInstantPath(
+ metaClient.getStorage(),
+ new StoragePath(metadataTableBasePath, METAFOLDER_NAME),
+ metadataCompactionInstant.get(), HoodieTimeline.COMMIT_ACTION)
.toUri());
FileCreateUtils.renameFileToTemp(metaFilePath, metadataCompactionInstant.get());
@@ -1026,9 +1040,11 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
client.rollback(newCommitTime3);
// mimicing crash or making an inflight in metadata table.
- Path toDelete = HoodieTestUtils
- .getCompleteInstantPath(metaClient.getFs(), new Path(metaClient.getMetaPath() + "/metadata/.hoodie/"), newCommitTime2, HoodieTimeline.DELTA_COMMIT_ACTION);
- metaClient.getFs().delete(toDelete);
+ StoragePath toDelete = HoodieTestUtils.getCompleteInstantPath(
+ metaClient.getStorage(),
+ new StoragePath(metaClient.getMetaPath() + "/metadata" + "/" + ".hoodie/"),
+ newCommitTime2, HoodieTimeline.DELTA_COMMIT_ACTION);
+ metaClient.getStorage().deleteDirectory(toDelete);
// re-ingest w/ same commit time.
records = dataGen.generateUniqueUpdates(newCommitTime3, 20);
@@ -1037,16 +1053,22 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
client.commit(newCommitTime3, writeStatuses);
// collect all commit meta files from metadata table.
- FileStatus[] metaFiles = metaClient.getFs().listStatus(new Path(metaClient.getMetaPath() + "/metadata/.hoodie"));
- List<FileStatus> commit3Files = Arrays.stream(metaFiles).filter(fileStatus ->
- fileStatus.getPath().getName().contains(newCommitTime3)
- && fileStatus.getPath().getName().contains(HoodieTimeline.DELTA_COMMIT_ACTION)).collect(Collectors.toList());
- List<FileStatus> rollbackFiles = Arrays.stream(metaFiles).filter(fileStatus ->
- fileStatus.getPath().getName().endsWith("." + HoodieTimeline.ROLLBACK_ACTION)).collect(Collectors.toList());
+ List<StoragePathInfo> metaFiles = metaClient.getStorage()
+ .listDirectEntries(new StoragePath(metaClient.getMetaPath() + "/metadata/.hoodie"));
+ List<StoragePathInfo> commit3Files = metaFiles.stream()
+ .filter(pathInfo ->
+ pathInfo.getPath().getName().contains(newCommitTime3)
+ && pathInfo.getPath().getName().contains(HoodieTimeline.DELTA_COMMIT_ACTION))
+ .collect(Collectors.toList());
+ List<StoragePathInfo> rollbackFiles = metaFiles.stream()
+ .filter(pathInfo ->
+ pathInfo.getPath().getName().endsWith("." + HoodieTimeline.ROLLBACK_ACTION))
+ .collect(Collectors.toList());
// ensure commit2's delta commit in MDT has last mod time > the actual rollback for previous failed commit i.e. commit2.
// if rollback wasn't eager, rollback's last mod time will be lower than the commit3'd delta commit last mod time.
- assertTrue(commit3Files.get(0).getModificationTime() > rollbackFiles.get(0).getModificationTime());
+ assertTrue(
+ commit3Files.get(0).getModificationTime() > rollbackFiles.get(0).getModificationTime());
}
}
@@ -1120,22 +1142,26 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertTrue(mdtTimeline.getCommitsTimeline().containsInstant(commit));
// Transition the last commit to inflight in DT
- deleteMetaFile(metaClient.getFs(), basePath, commit, COMMIT_EXTENSION);
+ deleteMetaFile(metaClient.getStorage(), basePath, commit, COMMIT_EXTENSION);
// Remove the last commit and written data files in MDT
- List<String> dataFiles = lastCommitMetadataWithValidData.getRight().getWriteStats().stream().map(
- HoodieWriteStat::getPath).collect(Collectors.toList());
+ List<String> dataFiles =
+ lastCommitMetadataWithValidData.getRight().getWriteStats().stream().map(
+ HoodieWriteStat::getPath).collect(Collectors.toList());
for (String relativeFilePath : dataFiles) {
- deleteFileFromDfs(metaClient.getFs(), mdtBasePath + "/" + relativeFilePath);
+ deleteFileFromStorage(metaClient.getStorage(), mdtBasePath + "/" + relativeFilePath);
}
- deleteMetaFile(metaClient.getFs(), mdtBasePath, commit, DELTA_COMMIT_EXTENSION);
- deleteMetaFile(metaClient.getFs(), mdtBasePath, commit, DELTA_COMMIT_EXTENSION + INFLIGHT_EXTENSION);
- deleteMetaFile(metaClient.getFs(), mdtBasePath, commit, DELTA_COMMIT_EXTENSION + REQUESTED_EXTENSION);
+ deleteMetaFile(metaClient.getStorage(), mdtBasePath, commit, DELTA_COMMIT_EXTENSION);
+ deleteMetaFile(metaClient.getStorage(), mdtBasePath, commit,
+ DELTA_COMMIT_EXTENSION + INFLIGHT_EXTENSION);
+ deleteMetaFile(metaClient.getStorage(), mdtBasePath, commit,
+ DELTA_COMMIT_EXTENSION + REQUESTED_EXTENSION);
// Transition the second init commit for record_index partition to inflight in MDT
- deleteMetaFile(metaClient.getFs(), mdtBasePath, mdtInitCommit2, DELTA_COMMIT_EXTENSION);
+ deleteMetaFile(
+ metaClient.getStorage(), mdtBasePath, mdtInitCommit2, DELTA_COMMIT_EXTENSION);
metaClient.getTableConfig().setMetadataPartitionState(
metaClient, MetadataPartitionType.RECORD_INDEX.getPartitionPath(), false);
metaClient.getTableConfig().setMetadataPartitionsInflight(
@@ -1148,21 +1174,27 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertTrue(mdtTimeline.lastInstant().get().isInflight());
}
- public static void deleteFileFromDfs(FileSystem fs, String targetPath) throws IOException {
- if (fs.exists(new Path(targetPath))) {
- fs.delete(new Path(targetPath), true);
+ public static void deleteFileFromStorage(HoodieStorage storage, String targetPath)
+ throws IOException {
+ if (storage.exists(new StoragePath(targetPath))) {
+ storage.deleteFile(new StoragePath(targetPath));
}
}
- public static void deleteMetaFile(FileSystem fs, String basePath, String instantTime, String suffix) throws IOException {
+ public static void deleteMetaFile(HoodieStorage storage,
+ String basePath,
+ String instantTime,
+ String suffix) throws IOException {
if (suffix.contains(REQUESTED_EXTENSION) || suffix.contains(INFLIGHT_EXTENSION)) {
String targetPath = basePath + "/" + METAFOLDER_NAME + "/" + instantTime + suffix;
- deleteFileFromDfs(fs, targetPath);
+ deleteFileFromStorage(storage, targetPath);
} else {
try {
- Path completeInstantPath = HoodieTestUtils.getCompleteInstantPath(fs,
- new Path(basePath + "/" + METAFOLDER_NAME), instantTime, suffix.replaceFirst(".", ""));
- deleteFileFromDfs(fs, completeInstantPath.toString());
+ StoragePath completeInstantPath = HoodieTestUtils.getCompleteInstantPath(
+ storage,
+ new StoragePath(basePath + "/" + METAFOLDER_NAME),
+ instantTime, suffix.replaceFirst(".", ""));
+ deleteFileFromStorage(storage, completeInstantPath.toString());
} catch (Exception e) {
// Ignore failure if file doesn't exist
}
@@ -1300,19 +1332,23 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
*/
private void verifyMetadataRawRecords(HoodieTable table, List<HoodieLogFile> logFiles, boolean enableMetaFields) throws IOException {
for (HoodieLogFile logFile : logFiles) {
- FileStatus[] fsStatus = fs.listStatus(logFile.getPath());
- MessageType writerSchemaMsg = TableSchemaResolver.readSchemaFromLogFile(fs, logFile.getPath());
+ List<StoragePathInfo> pathInfoList = storage.listDirectEntries(logFile.getPath());
+ MessageType writerSchemaMsg =
+ TableSchemaResolver.readSchemaFromLogFile(storage, logFile.getPath());
if (writerSchemaMsg == null) {
// not a data block
continue;
}
Schema writerSchema = new AvroSchemaConverter().convert(writerSchemaMsg);
- try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(fs, new HoodieLogFile(fsStatus[0].getPath()), writerSchema)) {
+ try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(storage,
+ new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) {
while (logFileReader.hasNext()) {
HoodieLogBlock logBlock = logFileReader.next();
if (logBlock instanceof HoodieDataBlock) {
- try (ClosableIterator<HoodieRecord<IndexedRecord>> recordItr = ((HoodieDataBlock) logBlock).getRecordIterator(HoodieRecordType.AVRO)) {
+ try (
+ ClosableIterator<HoodieRecord<IndexedRecord>> recordItr = ((HoodieDataBlock) logBlock).getRecordIterator(
+ HoodieRecordType.AVRO)) {
recordItr.forEachRemaining(indexRecord -> {
final GenericRecord record = (GenericRecord) indexRecord.getData();
if (enableMetaFields) {
@@ -1355,7 +1391,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
schema = HoodieAvroUtils.addMetadataFields(schema);
}
HoodieMetadataLogRecordReader logRecordReader = HoodieMetadataLogRecordReader.newBuilder()
- .withFileSystem(metadataMetaClient.getFs())
+ .withStorage(metadataMetaClient.getStorage())
.withBasePath(metadataMetaClient.getBasePath())
.withLogFilePaths(logFilePaths)
.withLatestInstantTime(latestCommitTimestamp)
@@ -1392,7 +1428,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
HoodieFileReaderFactory.getReaderFactory(HoodieRecordType.AVRO).getFileReader(
- table.getConfig(), context.getHadoopConf().get(), new Path(baseFile.getPath()));
+ table.getConfig(), context.getHadoopConf().get(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
records.forEach(entry -> {
if (enableMetaFields) {
@@ -1653,28 +1689,34 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
String newCommitTime = client.createNewInstantTime();
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 100);
client.startCommitWithTime(newCommitTime);
- List<WriteStatus> writeStatuses = client.insert(jsc.parallelize(records, 1), newCommitTime).collect();
+ List<WriteStatus> writeStatuses =
+ client.insert(jsc.parallelize(records, 1), newCommitTime).collect();
assertNoWriteErrors(writeStatuses);
validateMetadata(client);
// Metadata table should exist
- final Path metadataTablePath = new Path(getMetadataTableBasePath(writeConfig.getBasePath()));
- assertTrue(fs.exists(metadataTablePath));
+ final StoragePath metadataTablePath =
+ new StoragePath(getMetadataTableBasePath(writeConfig.getBasePath()));
+ assertTrue(storage.exists(metadataTablePath));
metaClient = HoodieTableMetaClient.reload(metaClient);
assertTrue(metaClient.getTableConfig().isMetadataTableAvailable());
// File groups should be created as in the config
HoodieBackedTableMetadata metadataReader = (HoodieBackedTableMetadata) metadata(client);
- assertEquals(HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataReader.getMetadataMetaClient(), Option.empty(),
+ assertEquals(HoodieTableMetadataUtil.getPartitionLatestFileSlices(
+ metadataReader.getMetadataMetaClient(), Option.empty(),
MetadataPartitionType.FILES.getPartitionPath()).size(), 1);
- assertEquals(HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataReader.getMetadataMetaClient(), Option.empty(),
+ assertEquals(HoodieTableMetadataUtil.getPartitionLatestFileSlices(
+ metadataReader.getMetadataMetaClient(), Option.empty(),
MetadataPartitionType.RECORD_INDEX.getPartitionPath()).size(), 5);
}
// remove the MDT partition from dataset to simulate failed bootstrap
Properties updateProperties = new Properties();
updateProperties.setProperty(HoodieTableConfig.TABLE_METADATA_PARTITIONS.key(), "");
- HoodieTableConfig.update(fs, new Path(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME),
+ HoodieTableConfig.update(
+ storage,
+ new StoragePath(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME),
updateProperties);
metaClient = HoodieTableMetaClient.reload(metaClient);
@@ -1804,8 +1846,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
this.tableType = COPY_ON_WRITE;
initPath();
initSparkContexts("TestHoodieMetadata");
- initFileSystem();
- fs.mkdirs(new Path(basePath));
+ initHoodieStorage();
+ storage.createDirectory(new StoragePath(basePath));
initTimelineService();
initMetaClient(tableType);
initTestDataGenerator();
@@ -1941,7 +1983,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
String commit1 = client.createNewInstantTime();
List<HoodieRecord> records = dataGen.generateInserts(commit1, 20);
client.startCommitWithTime(commit1);
- List<WriteStatus> writeStatuses = client.bulkInsert(jsc.parallelize(records, 1), commit1).collect();
+ List<WriteStatus> writeStatuses =
+ client.bulkInsert(jsc.parallelize(records, 1), commit1).collect();
assertNoWriteErrors(writeStatuses);
// Write 2 (inserts)
@@ -1951,9 +1994,11 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
writeStatuses = client.insert(jsc.parallelize(records, 1), commit2).collect();
assertNoWriteErrors(writeStatuses);
// remove latest completed delta commit from MDT.
- Path toDelete = HoodieTestUtils
- .getCompleteInstantPath(metaClient.getFs(), new Path(metaClient.getMetaPath() + "/metadata/.hoodie/"), commit2, HoodieTimeline.DELTA_COMMIT_ACTION);
- metaClient.getFs().delete(toDelete);
+ StoragePath toDelete = HoodieTestUtils.getCompleteInstantPath(
+ metaClient.getStorage(),
+ new StoragePath(metaClient.getMetaPath() + "/metadata/.hoodie/"),
+ commit2, HoodieTimeline.DELTA_COMMIT_ACTION);
+ metaClient.getStorage().deleteDirectory(toDelete);
// Write 3 (updates)
client.close();
@@ -1965,20 +2010,29 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertNoWriteErrors(writeStatuses);
// ensure that 000003 is after rollback of the partially failed 2nd commit.
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setBasePath(metaClient.getMetaPath() + "/metadata/").setConf(metaClient.getHadoopConf()).build();
- HoodieInstant rollbackInstant = metadataMetaClient.getActiveTimeline().getRollbackTimeline().getInstants().get(0);
+ HoodieTableMetaClient metadataMetaClient =
+ HoodieTableMetaClient.builder().setBasePath(metaClient.getMetaPath() + "/metadata/")
+ .setConf(metaClient.getHadoopConf()).build();
+ HoodieInstant rollbackInstant =
+ metadataMetaClient.getActiveTimeline().getRollbackTimeline().getInstants().get(0);
// collect all commit meta files from metadata table.
- FileStatus[] metaFiles = metaClient.getFs().listStatus(new Path(metaClient.getMetaPath() + "/metadata/.hoodie"));
- List<FileStatus> commit3Files = Arrays.stream(metaFiles).filter(fileStatus ->
- fileStatus.getPath().getName().contains(commit3)
- && fileStatus.getPath().getName().contains(HoodieTimeline.DELTA_COMMIT_ACTION)).collect(Collectors.toList());
- List<FileStatus> rollbackFiles = Arrays.stream(metaFiles).filter(fileStatus ->
- fileStatus.getPath().getName().equals(rollbackInstant.getFileName())).collect(Collectors.toList());
+ List<StoragePathInfo> metaFiles = metaClient.getStorage()
+ .listDirectEntries(new StoragePath(metaClient.getMetaPath() + "/metadata/.hoodie"));
+ List<StoragePathInfo> commit3Files = metaFiles.stream()
+ .filter(pathInfo ->
+ pathInfo.getPath().getName().contains(commit3)
+ && pathInfo.getPath().getName().contains(HoodieTimeline.DELTA_COMMIT_ACTION))
+ .collect(Collectors.toList());
+ List<StoragePathInfo> rollbackFiles = metaFiles.stream()
+ .filter(pathInfo ->
+ pathInfo.getPath().getName().equals(rollbackInstant.getFileName()))
+ .collect(Collectors.toList());
// ensure commit3's delta commit in MDT has last mod time > the actual rollback for previous failed commit i.e. commit2.
// if rollback wasn't eager, rollback's last mod time will be lower than the commit3'd delta commit last mod time.
- assertTrue(commit3Files.get(0).getModificationTime() > rollbackFiles.get(0).getModificationTime());
+ assertTrue(
+ commit3Files.get(0).getModificationTime() > rollbackFiles.get(0).getModificationTime());
client.close();
}
@@ -2529,37 +2583,46 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
// Metadata table should have been bootstrapped
- assertTrue(fs.exists(new Path(metadataTableBasePath)), "Metadata table should exist");
- FileStatus oldStatus = fs.getFileStatus(new Path(metadataTableBasePath));
+ assertTrue(storage.exists(new StoragePath(metadataTableBasePath)),
+ "Metadata table should exist");
+ StoragePathInfo oldInfo = storage.getPathInfo(new StoragePath(metadataTableBasePath));
// set hoodie.table.version to 2 in hoodie.properties file
changeTableVersion(HoodieTableVersion.TWO);
// With next commit the table should be deleted (as part of upgrade) and then re-bootstrapped automatically
metaClient.reloadActiveTimeline();
- FileStatus prevStatus = fs.getFileStatus(new Path(metadataTableBasePath));
- try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, getWriteConfig(true, true))) {
+ StoragePathInfo prevInfo = storage.getPathInfo(new StoragePath(metadataTableBasePath));
+ try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext,
+ getWriteConfig(true, true))) {
commitTimestamp = client.createNewInstantTime();
records = dataGen.generateInserts(commitTimestamp, 5);
client.startCommitWithTime(commitTimestamp);
writeStatuses = client.bulkInsert(jsc.parallelize(records, 1), commitTimestamp).collect();
assertNoWriteErrors(writeStatuses);
}
- assertTrue(fs.exists(new Path(metadataTableBasePath)), "Metadata table should exist");
- FileStatus currentStatus = fs.getFileStatus(new Path(metadataTableBasePath));
- assertTrue(currentStatus.getModificationTime() > prevStatus.getModificationTime());
+ assertTrue(storage.exists(new StoragePath(metadataTableBasePath)),
+ "Metadata table should exist");
+ StoragePathInfo currentInfo =
+ storage.getPathInfo(new StoragePath(metadataTableBasePath));
+ assertTrue(currentInfo.getModificationTime() > prevInfo.getModificationTime());
initMetaClient();
- assertEquals(metaClient.getTableConfig().getTableVersion().versionCode(), HoodieTableVersion.current().versionCode());
- assertTrue(fs.exists(new Path(metadataTableBasePath)), "Metadata table should exist");
- FileStatus newStatus = fs.getFileStatus(new Path(metadataTableBasePath));
- assertTrue(oldStatus.getModificationTime() < newStatus.getModificationTime());
+ assertEquals(metaClient.getTableConfig().getTableVersion().versionCode(),
+ HoodieTableVersion.current().versionCode());
+ assertTrue(storage.exists(new StoragePath(metadataTableBasePath)),
+ "Metadata table should exist");
+ StoragePathInfo newInfo = storage.getPathInfo(new StoragePath(metadataTableBasePath));
+ assertTrue(oldInfo.getModificationTime() < newInfo.getModificationTime());
// Test downgrade by running the downgrader
- new UpgradeDowngrade(metaClient, writeConfig, context, SparkUpgradeDowngradeHelper.getInstance()).run(HoodieTableVersion.TWO, null);
+ new UpgradeDowngrade(metaClient, writeConfig, context,
+ SparkUpgradeDowngradeHelper.getInstance()).run(HoodieTableVersion.TWO, null);
metaClient = HoodieTableMetaClient.reload(metaClient);
- assertEquals(HoodieTableVersion.TWO.versionCode(), metaClient.getTableConfig().getTableVersion().versionCode());
- assertFalse(fs.exists(new Path(metadataTableBasePath)), "Metadata table should not exist");
+ assertEquals(HoodieTableVersion.TWO.versionCode(),
+ metaClient.getTableConfig().getTableVersion().versionCode());
+ assertFalse(storage.exists(new StoragePath(metadataTableBasePath)),
+ "Metadata table should not exist");
}
/**
@@ -2596,8 +2659,9 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
// Metadata table should have been bootstrapped
- assertTrue(fs.exists(new Path(metadataTableBasePath)), "Metadata table should exist");
- FileStatus oldStatus = fs.getFileStatus(new Path(metadataTableBasePath));
+ assertTrue(storage.exists(new StoragePath(metadataTableBasePath)),
+ "Metadata table should exist");
+ StoragePathInfo oldInfo = storage.getPathInfo(new StoragePath(metadataTableBasePath));
// trigger partial commit
metaClient.reloadActiveTimeline();
@@ -2629,10 +2693,12 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
initMetaClient();
- assertEquals(metaClient.getTableConfig().getTableVersion().versionCode(), HoodieTableVersion.current().versionCode());
- assertTrue(fs.exists(new Path(metadataTableBasePath)), "Metadata table should exist");
- FileStatus newStatus = fs.getFileStatus(new Path(metadataTableBasePath));
- assertTrue(oldStatus.getModificationTime() < newStatus.getModificationTime());
+ assertEquals(metaClient.getTableConfig().getTableVersion().versionCode(),
+ HoodieTableVersion.current().versionCode());
+ assertTrue(storage.exists(new StoragePath(metadataTableBasePath)),
+ "Metadata table should exist");
+ StoragePathInfo newInfo = storage.getPathInfo(new StoragePath(metadataTableBasePath));
+ assertTrue(oldInfo.getModificationTime() < newInfo.getModificationTime());
}
/**
@@ -2669,9 +2735,11 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// There is no way to simulate failed commit on the main dataset, hence we simply delete the completed
// instant so that only the inflight is left over.
- String commitInstantFileName = metaClient.getActiveTimeline().getReverseOrderedInstants().findFirst().get().getFileName();
- assertTrue(fs.delete(new Path(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME,
- commitInstantFileName), false));
+ String commitInstantFileName =
+ metaClient.getActiveTimeline().getReverseOrderedInstants().findFirst().get()
+ .getFileName();
+ assertTrue(storage.deleteFile(new StoragePath(
+ basePath + StoragePath.SEPARATOR + METAFOLDER_NAME, commitInstantFileName)));
}
try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext,
@@ -2721,9 +2789,9 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// delete the metadata table partitions to check, whether rollback of pending commit succeeds and
// metadata table partitions are rebootstrapped.
metadataWriter.dropMetadataPartitions(Arrays.asList(MetadataPartitionType.RECORD_INDEX, FILES));
- assertFalse(fs.exists(new Path(getMetadataTableBasePath(basePath)
- + StoragePath.SEPARATOR + FILES.getPartitionPath())));
- assertFalse(fs.exists(new Path(getMetadataTableBasePath(basePath)
+ assertFalse(storage.exists(new StoragePath(
+ getMetadataTableBasePath(basePath) + StoragePath.SEPARATOR + FILES.getPartitionPath())));
+ assertFalse(storage.exists(new StoragePath(getMetadataTableBasePath(basePath)
+ StoragePath.SEPARATOR + MetadataPartitionType.RECORD_INDEX.getPartitionPath())));
metaClient = HoodieTableMetaClient.reload(metaClient);
@@ -2741,13 +2809,14 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
writeStatuses = client.insert(jsc.parallelize(records, 1), commitTime).collect();
}
assertNoWriteErrors(writeStatuses);
- assertTrue(fs.exists(new Path(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME)));
+ assertTrue(storage.exists(new StoragePath(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME)));
metaClient = HoodieTableMetaClient.reload(metaClient);
- assertFalse(metaClient.getActiveTimeline().filterCompletedInstants().filterCompletedInstants().findInstantsAfterOrEquals(commitTime, 1).empty());
+ assertFalse(metaClient.getActiveTimeline().filterCompletedInstants().filterCompletedInstants()
+ .findInstantsAfterOrEquals(commitTime, 1).empty());
- assertTrue(fs.exists(new Path(getMetadataTableBasePath(basePath)
- + StoragePath.SEPARATOR + FILES.getPartitionPath())));
- assertTrue(fs.exists(new Path(getMetadataTableBasePath(basePath)
+ assertTrue(storage.exists(new StoragePath(
+ getMetadataTableBasePath(basePath) + StoragePath.SEPARATOR + FILES.getPartitionPath())));
+ assertTrue(storage.exists(new StoragePath(getMetadataTableBasePath(basePath)
+ StoragePath.SEPARATOR + MetadataPartitionType.RECORD_INDEX.getPartitionPath())));
}
@@ -2766,16 +2835,19 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
String newCommitTime = client.createNewInstantTime();
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 1);
client.startCommitWithTime(newCommitTime);
- List<WriteStatus> writeStatuses = client.insert(jsc.parallelize(records, 1), newCommitTime).collect();
+ List<WriteStatus> writeStatuses =
+ client.insert(jsc.parallelize(records, 1), newCommitTime).collect();
assertNoWriteErrors(writeStatuses);
validateMetadata(client);
}
- final Path metadataTablePath = new Path(getMetadataTableBasePath(writeConfig.getBasePath()));
- assertTrue(fs.exists(metadataTablePath), "metadata table should exist.");
+ final StoragePath metadataTablePath = new StoragePath(
+ getMetadataTableBasePath(writeConfig.getBasePath()));
+ assertTrue(storage.exists(metadataTablePath), "metadata table should exist.");
deleteMetadataTable(metaClient, context, false);
- assertFalse(fs.exists(metadataTablePath), "metadata table should not exist after being deleted.");
+ assertFalse(storage.exists(metadataTablePath),
+ "metadata table should not exist after being deleted.");
writeConfig = getWriteConfigBuilder(true, true, false).build();
try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, writeConfig)) {
@@ -2783,12 +2855,13 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
String newCommitTime = client.createNewInstantTime();
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 1);
client.startCommitWithTime(newCommitTime);
- List<WriteStatus> writeStatuses = client.insert(jsc.parallelize(records, 1), newCommitTime).collect();
+ List<WriteStatus> writeStatuses =
+ client.insert(jsc.parallelize(records, 1), newCommitTime).collect();
assertNoWriteErrors(writeStatuses);
}
// Metadata table is recreated, during bootstrapping of metadata table.
- assertTrue(fs.exists(metadataTablePath));
+ assertTrue(storage.exists(metadataTablePath));
}
/**
@@ -2888,9 +2961,11 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// There is no way to simulate failed commit on the main dataset, hence we simply delete the completed
// instant so that only the inflight is left over.
- String commitInstantFileName = metaClient.getActiveTimeline().getReverseOrderedInstants().findFirst().get().getFileName();
- assertTrue(fs.delete(new Path(basePath + StoragePath.SEPARATOR + METAFOLDER_NAME,
- commitInstantFileName), false));
+ String commitInstantFileName =
+ metaClient.getActiveTimeline().getReverseOrderedInstants().findFirst().get()
+ .getFileName();
+ assertTrue(storage.deleteFile(new StoragePath(
+ basePath + StoragePath.SEPARATOR + METAFOLDER_NAME, commitInstantFileName)));
}
try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext,
@@ -3058,11 +3133,13 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// To test duplicates during bootstrap, insert duplicates in the first batch.
recordsFirstBatch.addAll(insertRecords);
client.startCommitWithTime(firstCommitTime);
- List<WriteStatus> writeStatuses = client.insert(jsc.parallelize(recordsFirstBatch, 1), firstCommitTime).collect();
+ List<WriteStatus> writeStatuses =
+ client.insert(jsc.parallelize(recordsFirstBatch, 1), firstCommitTime).collect();
assertNoWriteErrors(writeStatuses);
commitTimestamps.add(firstCommitTime);
}
- assertEquals(false, fs.exists(new Path(metaClient.getMetaPath(), "metadata/record_index")));
+ assertEquals(false,
+ storage.exists(new StoragePath(metaClient.getMetaPath(), "metadata/record_index")));
// bootstrap record index
customConfig = getWriteConfigBuilder(false, true, false)
@@ -3120,17 +3197,24 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// 1 partition should be cleaned
assertEquals(cleanMetadata.getPartitionMetadata().size(), 1);
// 1 file cleaned
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getSuccessDeleteFiles().size(), 1);
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getFailedDeleteFiles().size(), 0);
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getDeletePathPatterns().size(), 1);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getSuccessDeleteFiles().size(), 1);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getFailedDeleteFiles().size(), 0);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getDeletePathPatterns().size(), 1);
// To simulate failed clean on the main dataset, we will delete the completed clean instant
- String cleanInstantFileName = metaClient.reloadActiveTimeline().getCleanerTimeline().filterCompletedInstants()
- .getReverseOrderedInstants().findFirst().get().getFileName();
- assertTrue(fs.delete(new Path(basePath + StoragePath.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME,
- cleanInstantFileName), false));
- assertEquals(metaClient.reloadActiveTimeline().getCleanerTimeline().filterInflights().countInstants(), 1);
- assertEquals(metaClient.reloadActiveTimeline().getCleanerTimeline().filterCompletedInstants().countInstants(), 0);
+ String cleanInstantFileName =
+ metaClient.reloadActiveTimeline().getCleanerTimeline().filterCompletedInstants()
+ .getReverseOrderedInstants().findFirst().get().getFileName();
+ assertTrue(storage.deleteFile(new StoragePath(
+ basePath + StoragePath.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME, cleanInstantFileName)));
+ assertEquals(
+ metaClient.reloadActiveTimeline().getCleanerTimeline().filterInflights().countInstants(),
+ 1);
+ assertEquals(metaClient.reloadActiveTimeline().getCleanerTimeline().filterCompletedInstants()
+ .countInstants(), 0);
// Initiate another clean. The previous leftover clean will be attempted and no other clean will be scheduled.
String newCleanInstantTime = "00" + index++;
@@ -3139,9 +3223,12 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// 1 partition should be cleaned
assertEquals(cleanMetadata.getPartitionMetadata().size(), 1);
// 1 file cleaned but was already deleted so will be a failed delete
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getSuccessDeleteFiles().size(), 0);
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getFailedDeleteFiles().size(), 1);
- assertEquals(cleanMetadata.getPartitionMetadata().get(partition).getDeletePathPatterns().size(), 1);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getSuccessDeleteFiles().size(), 0);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getFailedDeleteFiles().size(), 1);
+ assertEquals(
+ cleanMetadata.getPartitionMetadata().get(partition).getDeletePathPatterns().size(), 1);
validateMetadata(client);
}
@@ -3464,39 +3551,47 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// Files within each partition should match
HoodieTable table = HoodieSparkTable.create(config, engineContext);
TableFileSystemView tableView = table.getHoodieView();
- List<String> fullPartitionPaths = fsPartitions.stream().map(partition -> basePath + "/" + partition).collect(Collectors.toList());
- Map<String, FileStatus[]> partitionToFilesMap = tableMetadata.getAllFilesInPartitions(fullPartitionPaths);
+ List<String> fullPartitionPaths =
+ fsPartitions.stream().map(partition -> basePath + "/" + partition)
+ .collect(Collectors.toList());
+ Map<String, List<StoragePathInfo>> partitionToFilesMap =
+ tableMetadata.getAllFilesInPartitions(fullPartitionPaths);
assertEquals(fsPartitions.size(), partitionToFilesMap.size());
fsPartitions.forEach(partition -> {
try {
- Path partitionPath;
+ StoragePath partitionPath;
if (partition.equals("")) {
// Should be the non-partitioned case
- partitionPath = new Path(basePath);
+ partitionPath = new StoragePath(basePath);
} else {
- partitionPath = new Path(basePath, partition);
+ partitionPath = new StoragePath(basePath, partition);
}
- FileStatus[] fsStatuses = FSUtils.getAllDataFilesInPartition(fs, partitionPath);
+ List<StoragePathInfo> pathInfoList =
+ FSUtils.getAllDataFilesInPartition(storage, partitionPath);
if (ignoreFilesWithCommit.isPresent()) {
- fsStatuses = Arrays.stream(fsStatuses).filter(fileStatus -> !fileStatus.getPath().getName().contains(ignoreFilesWithCommit.get()))
- .collect(Collectors.toList()).toArray(new FileStatus[0]);
+ pathInfoList = pathInfoList.stream()
+ .filter(pathInfo ->
+ !pathInfo.getPath().getName().contains(ignoreFilesWithCommit.get()))
+ .collect(Collectors.toList());
}
- FileStatus[] metaStatuses = tableMetadata.getAllFilesInPartition(partitionPath);
- List<String> fsFileNames = Arrays.stream(fsStatuses)
+ List<StoragePathInfo> metaFilesList = tableMetadata.getAllFilesInPartition(partitionPath);
+ List<String> fsFileNames = pathInfoList.stream()
.map(s -> s.getPath().getName()).collect(Collectors.toList());
- List<String> metadataFilenames = Arrays.stream(metaStatuses)
+ List<String> metadataFilenames = metaFilesList.stream()
.map(s -> s.getPath().getName()).collect(Collectors.toList());
Collections.sort(fsFileNames);
Collections.sort(metadataFilenames);
- assertEquals(fsStatuses.length, partitionToFilesMap.get(partitionPath.toString()).length, "Files within partition " + partition + " should match");
+ assertEquals(pathInfoList.size(), partitionToFilesMap.get(partitionPath.toString()).size(),
+ "Files within partition " + partition + " should match");
// File sizes should be valid
- Arrays.stream(metaStatuses).forEach(s -> assertTrue(s.getLen() > 0));
+ metaFilesList.stream().forEach(s -> assertTrue(s.getLength() > 0));
- if ((fsFileNames.size() != metadataFilenames.size()) || (!fsFileNames.equals(metadataFilenames))) {
+ if ((fsFileNames.size() != metadataFilenames.size())
+ || (!fsFileNames.equals(metadataFilenames))) {
LOG.info("*** File system listing = " + Arrays.toString(fsFileNames.toArray()));
LOG.info("*** Metadata listing = " + Arrays.toString(metadataFilenames.toArray()));
@@ -3513,26 +3608,34 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
// Block sizes should be valid
- Arrays.stream(metaStatuses).forEach(s -> assertTrue(s.getBlockSize() > 0));
- List<Long> fsBlockSizes = Arrays.stream(fsStatuses).map(FileStatus::getBlockSize).collect(Collectors.toList());
+ metaFilesList.forEach(s -> assertTrue(s.getBlockSize() > 0));
+ List<Long> fsBlockSizes = pathInfoList.stream().map(StoragePathInfo::getBlockSize).collect(Collectors.toList());
Collections.sort(fsBlockSizes);
- List<Long> metadataBlockSizes = Arrays.stream(metaStatuses).map(FileStatus::getBlockSize).collect(Collectors.toList());
+ List<Long> metadataBlockSizes = metaFilesList.stream().map(StoragePathInfo::getBlockSize).collect(Collectors.toList());
Collections.sort(metadataBlockSizes);
assertEquals(fsBlockSizes, metadataBlockSizes);
- assertEquals(fsFileNames.size(), metadataFilenames.size(), "Files within partition " + partition + " should match");
- assertTrue(fsFileNames.equals(metadataFilenames), "Files within partition " + partition + " should match");
+ assertEquals(fsFileNames.size(), metadataFilenames.size(),
+ "Files within partition " + partition + " should match");
+ assertTrue(fsFileNames.equals(metadataFilenames),
+ "Files within partition " + partition + " should match");
// FileSystemView should expose the same data
- List<HoodieFileGroup> fileGroups = tableView.getAllFileGroups(partition).collect(Collectors.toList());
- fileGroups.addAll(tableView.getAllReplacedFileGroups(partition).collect(Collectors.toList()));
-
- fileGroups.forEach(g -> LoggerFactory.getLogger(TestHoodieBackedMetadata.class).info(g.toString()));
- fileGroups.forEach(g -> g.getAllBaseFiles().forEach(b -> LoggerFactory.getLogger(TestHoodieBackedMetadata.class).info(b.toString())));
- fileGroups.forEach(g -> g.getAllFileSlices().forEach(s -> LoggerFactory.getLogger(TestHoodieBackedMetadata.class).info(s.toString())));
+ List<HoodieFileGroup> fileGroups =
+ tableView.getAllFileGroups(partition).collect(Collectors.toList());
+ fileGroups.addAll(
+ tableView.getAllReplacedFileGroups(partition).collect(Collectors.toList()));
+
+ fileGroups.forEach(
+ g -> LoggerFactory.getLogger(TestHoodieBackedMetadata.class).info(g.toString()));
+ fileGroups.forEach(g -> g.getAllBaseFiles().forEach(
+ b -> LoggerFactory.getLogger(TestHoodieBackedMetadata.class).info(b.toString())));
+ fileGroups.forEach(g -> g.getAllFileSlices().forEach(
+ s -> LoggerFactory.getLogger(TestHoodieBackedMetadata.class).info(s.toString())));
long numFiles = fileGroups.stream()
- .mapToLong(g -> g.getAllBaseFiles().count() + g.getAllFileSlices().mapToLong(s -> s.getLogFiles().count()).sum())
+ .mapToLong(g -> g.getAllBaseFiles().count()
+ + g.getAllFileSlices().mapToLong(s -> s.getLogFiles().count()).sum())
.sum();
assertEquals(metadataFilenames.size(), numFiles);
} catch (IOException e) {
@@ -3595,25 +3698,32 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
private void verifyMetadataColumnStatsRecords(List<HoodieLogFile> logFiles) throws IOException {
for (HoodieLogFile logFile : logFiles) {
- FileStatus[] fsStatus = fs.listStatus(logFile.getPath());
- MessageType writerSchemaMsg = TableSchemaResolver.readSchemaFromLogFile(fs, logFile.getPath());
+ List<StoragePathInfo> pathInfoList = storage.listDirectEntries(logFile.getPath());
+ MessageType writerSchemaMsg =
+ TableSchemaResolver.readSchemaFromLogFile(storage, logFile.getPath());
if (writerSchemaMsg == null) {
// not a data block
continue;
}
Schema writerSchema = new AvroSchemaConverter().convert(writerSchemaMsg);
- try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(fs, new HoodieLogFile(fsStatus[0].getPath()), writerSchema)) {
+ try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(storage,
+ new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) {
while (logFileReader.hasNext()) {
HoodieLogBlock logBlock = logFileReader.next();
if (logBlock instanceof HoodieDataBlock) {
- try (ClosableIterator<HoodieRecord<IndexedRecord>> recordItr = ((HoodieDataBlock) logBlock).getRecordIterator(HoodieRecordType.AVRO)) {
+ try (
+ ClosableIterator<HoodieRecord<IndexedRecord>> recordItr = ((HoodieDataBlock) logBlock).getRecordIterator(
+ HoodieRecordType.AVRO)) {
recordItr.forEachRemaining(indexRecord -> {
final GenericRecord record = (GenericRecord) indexRecord.getData();
- final GenericRecord colStatsRecord = (GenericRecord) record.get(HoodieMetadataPayload.SCHEMA_FIELD_ID_COLUMN_STATS);
+ final GenericRecord colStatsRecord =
+ (GenericRecord) record.get(HoodieMetadataPayload.SCHEMA_FIELD_ID_COLUMN_STATS);
assertNotNull(colStatsRecord);
- assertNotNull(colStatsRecord.get(HoodieMetadataPayload.COLUMN_STATS_FIELD_COLUMN_NAME));
- assertNotNull(colStatsRecord.get(HoodieMetadataPayload.COLUMN_STATS_FIELD_NULL_COUNT));
+ assertNotNull(
+ colStatsRecord.get(HoodieMetadataPayload.COLUMN_STATS_FIELD_COLUMN_NAME));
+ assertNotNull(
+ colStatsRecord.get(HoodieMetadataPayload.COLUMN_STATS_FIELD_NULL_COUNT));
/**
* TODO: some types of field may have null min/max as these statistics are only supported for primitive types
* assertNotNull(colStatsRecord.get(HoodieMetadataPayload.COLUMN_STATS_FIELD_MAX_VALUE));
@@ -3630,11 +3740,12 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
/**
* Returns the list of all files in the dataset by iterating over the metadata table.
*/
- private List<Path> getAllFiles(HoodieTableMetadata metadata) throws Exception {
- List<Path> allfiles = new LinkedList<>();
+ private List<StoragePath> getAllFiles(HoodieTableMetadata metadata) throws Exception {
+ List<StoragePath> allfiles = new LinkedList<>();
for (String partition : metadata.getAllPartitionPaths()) {
- for (FileStatus status : metadata.getAllFilesInPartition(new Path(basePath, partition))) {
- allfiles.add(status.getPath());
+ for (StoragePathInfo pathInfo :
+ metadata.getAllFilesInPartition(new StoragePath(basePath, partition))) {
+ allfiles.add(pathInfo.getPath());
}
}
@@ -3654,8 +3765,9 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
private void changeTableVersion(HoodieTableVersion version) throws IOException {
metaClient = HoodieTableMetaClient.reload(metaClient);
metaClient.getTableConfig().setTableVersion(version);
- Path propertyFile = new Path(metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE);
- try (OutputStream os = metaClient.getFs().create(propertyFile)) {
+ StoragePath propertyFile = new StoragePath(
+ metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE);
+ try (OutputStream os = metaClient.getStorage().create(propertyFile)) {
metaClient.getTableConfig().getProps().store(os, "");
}
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
index 8ca0d4e16a9..e3312e6431a 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
@@ -51,14 +51,14 @@ import org.apache.hudi.metadata.HoodieBackedTableMetadata;
import org.apache.hudi.metadata.HoodieMetadataLogRecordReader;
import org.apache.hudi.metadata.HoodieMetadataPayload;
import org.apache.hudi.metadata.HoodieTableMetadataKeyGenerator;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.schema.MessageType;
import org.junit.jupiter.params.ParameterizedTest;
@@ -144,12 +144,13 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
try {
downLatch.countDown();
downLatch.await();
- FileStatus[] files = tableMetadata.getAllFilesInPartition(new Path(finalPartition));
- if (files.length != 1) {
+ List<StoragePathInfo> files =
+ tableMetadata.getAllFilesInPartition(new StoragePath(finalPartition));
+ if (files.size() != 1) {
LOG.warn("Miss match data file numbers.");
throw new RuntimeException("Miss match data file numbers.");
}
- filesNumber.addAndGet(files.length);
+ filesNumber.addAndGet(files.size());
} catch (Exception e) {
LOG.warn("Catch Exception while reading data files from MDT.", e);
flag.compareAndSet(false, true);
@@ -185,13 +186,17 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
// Files within each partition should match
HoodieTable table = HoodieSparkTable.create(writeConfig, context);
TableFileSystemView tableView = table.getHoodieView();
- List<String> fullPartitionPaths = fsPartitions.stream().map(partition -> basePath + "/" + partition).collect(Collectors.toList());
- Map<String, FileStatus[]> partitionToFilesMap = tableMetadata.getAllFilesInPartitions(fullPartitionPaths);
+ List<String> fullPartitionPaths =
+ fsPartitions.stream().map(partition -> basePath + "/" + partition)
+ .collect(Collectors.toList());
+ Map<String, List<StoragePathInfo>> partitionToFilesMap =
+ tableMetadata.getAllFilesInPartitions(fullPartitionPaths);
assertEquals(fsPartitions.size(), partitionToFilesMap.size());
fsPartitions.forEach(partition -> {
try {
- validateFilesPerPartition(testTable, tableMetadata, tableView, partitionToFilesMap, partition);
+ validateFilesPerPartition(testTable, tableMetadata, tableView, partitionToFilesMap,
+ partition);
} catch (IOException e) {
fail("Exception should not be raised: " + e);
}
@@ -223,9 +228,9 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
init(tableType);
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(context,
writeConfig.getMetadataConfig(), writeConfig.getBasePath(), false);
- FileStatus[] allFilesInPartition =
- tableMetadata.getAllFilesInPartition(new Path(writeConfig.getBasePath() + "dummy"));
- assertEquals(allFilesInPartition.length, 0);
+ List<StoragePathInfo> allFilesInPartition = tableMetadata.getAllFilesInPartition(
+ new StoragePath(writeConfig.getBasePath() + "dummy"));
+ assertEquals(allFilesInPartition.size(), 0);
}
/**
@@ -387,7 +392,8 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
new HoodieLocalEngineContext(hadoopConf),
HoodieMetadataConfig.newBuilder().enable(true).build(),
basePath);
- return Arrays.stream(tableMetadata.getAllFilesInPartition(new Path(basePath, partition)))
+ return tableMetadata.getAllFilesInPartition(new StoragePath(basePath, partition))
+ .stream()
.map(status -> status.getPath().getName()).collect(Collectors.toSet());
}
@@ -452,19 +458,23 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
*/
private void verifyMetadataRawRecords(HoodieTable table, List<HoodieLogFile> logFiles) throws IOException {
for (HoodieLogFile logFile : logFiles) {
- FileStatus[] fsStatus = fs.listStatus(logFile.getPath());
- MessageType writerSchemaMsg = TableSchemaResolver.readSchemaFromLogFile(fs, logFile.getPath());
+ List<StoragePathInfo> pathInfoList = storage.listDirectEntries(logFile.getPath());
+ MessageType writerSchemaMsg =
+ TableSchemaResolver.readSchemaFromLogFile(storage, logFile.getPath());
if (writerSchemaMsg == null) {
// not a data block
continue;
}
Schema writerSchema = new AvroSchemaConverter().convert(writerSchemaMsg);
- try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(fs, new HoodieLogFile(fsStatus[0].getPath()), writerSchema)) {
+ try (HoodieLogFormat.Reader logFileReader = HoodieLogFormat.newReader(storage,
+ new HoodieLogFile(pathInfoList.get(0).getPath()), writerSchema)) {
while (logFileReader.hasNext()) {
HoodieLogBlock logBlock = logFileReader.next();
if (logBlock instanceof HoodieDataBlock) {
- try (ClosableIterator<HoodieRecord<IndexedRecord>> recordItr = ((HoodieDataBlock) logBlock).getRecordIterator(HoodieRecordType.AVRO)) {
+ try (
+ ClosableIterator<HoodieRecord<IndexedRecord>> recordItr = ((HoodieDataBlock) logBlock).getRecordIterator(
+ HoodieRecordType.AVRO)) {
recordItr.forEachRemaining(indexRecord -> {
final GenericRecord record = (GenericRecord) indexRecord.getData();
assertNull(record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD));
@@ -491,7 +501,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
private void verifyMetadataMergedRecords(HoodieTableMetaClient metadataMetaClient, List<String> logFilePaths, String latestCommitTimestamp) {
Schema schema = HoodieAvroUtils.addMetadataFields(HoodieMetadataRecord.getClassSchema());
HoodieMetadataLogRecordReader logRecordReader = HoodieMetadataLogRecordReader.newBuilder()
- .withFileSystem(metadataMetaClient.getFs())
+ .withStorage(metadataMetaClient.getStorage())
.withBasePath(metadataMetaClient.getBasePath())
.withLogFilePaths(logFilePaths)
.withLatestInstantTime(latestCommitTimestamp)
@@ -527,7 +537,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
HoodieAvroHFileReaderImplBase hoodieHFileReader = (HoodieAvroHFileReaderImplBase)
HoodieFileReaderFactory.getReaderFactory(HoodieRecordType.AVRO).getFileReader(
- table.getConfig(), context.getHadoopConf().get(), new Path(baseFile.getPath()));
+ table.getConfig(), context.getHadoopConf().get(), new StoragePath(baseFile.getPath()));
List<IndexedRecord> records = HoodieAvroHFileReaderImplBase.readAllRecords(hoodieHFileReader);
records.forEach(entry -> {
assertNull(((GenericRecord) entry).get(HoodieRecord.RECORD_KEY_METADATA_FIELD));
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index cff783ebd53..07ee1122c64 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -107,6 +107,8 @@ import org.apache.hudi.io.HoodieMergeHandle;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.KeyGenerator;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.hudi.table.HoodieSparkCopyOnWriteTable;
import org.apache.hudi.table.HoodieSparkTable;
@@ -120,7 +122,6 @@ import org.apache.hudi.testutils.HoodieClientTestUtils;
import org.apache.hudi.testutils.HoodieSparkWriteableTestTable;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.Dataset;
@@ -668,7 +669,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
for (int i = 0; i < fullPartitionPaths.length; i++) {
fullPartitionPaths[i] = String.format("%s/%s/*", basePath, dataGen.getPartitionPaths()[i]);
}
- assertEquals(200, HoodieClientTestUtils.read(jsc, basePath, sqlContext, fs, fullPartitionPaths).count(),
+ assertEquals(200,
+ HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths).count(),
"Must contain " + 200 + " records");
// Perform Delete again on upgraded dataset.
@@ -942,9 +944,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@Test
public void testPendingRestore() throws IOException {
- HoodieWriteConfig config = getConfigBuilder().withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()).build();
- Path completeRestoreFile = null;
- Path backupCompletedRestoreFile = null;
+ HoodieWriteConfig config = getConfigBuilder().withMetadataConfig(
+ HoodieMetadataConfig.newBuilder().enable(false).build()).build();
+ StoragePath completeRestoreFile = null;
+ StoragePath backupCompletedRestoreFile = null;
try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
final String commitTime1 = "001";
client.startCommitWithTime(commitTime1);
@@ -958,11 +961,16 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
client.restoreToInstant("001", false);
// remove completed restore instant from timeline to mimic pending restore.
- HoodieInstant restoreCompleted = metaClient.reloadActiveTimeline().getRestoreTimeline().filterCompletedInstants().getInstants().get(0);
- completeRestoreFile = new Path(config.getBasePath() + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + restoreCompleted.getFileName());
- backupCompletedRestoreFile = new Path(config.getBasePath() + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
- + restoreCompleted.getFileName() + ".backup");
- metaClient.getFs().rename(completeRestoreFile, backupCompletedRestoreFile);
+ HoodieInstant restoreCompleted =
+ metaClient.reloadActiveTimeline().getRestoreTimeline().filterCompletedInstants()
+ .getInstants().get(0);
+ completeRestoreFile = new StoragePath(
+ config.getBasePath() + "/" + HoodieTableMetaClient.METAFOLDER_NAME
+ + "/" + restoreCompleted.getFileName());
+ backupCompletedRestoreFile = new StoragePath(
+ config.getBasePath() + "/" + HoodieTableMetaClient.METAFOLDER_NAME
+ + "/" + restoreCompleted.getFileName() + ".backup");
+ metaClient.getStorage().rename(completeRestoreFile, backupCompletedRestoreFile);
}
try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
@@ -971,7 +979,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertThrows(IllegalArgumentException.class, () -> client.startCommitWithTime(commitTime2));
}
// add back the restore file.
- metaClient.getFs().rename(backupCompletedRestoreFile, completeRestoreFile);
+ metaClient.getStorage().rename(backupCompletedRestoreFile, completeRestoreFile);
// retrigger a new commit, should succeed.
@@ -1083,8 +1091,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
}
private Dataset<org.apache.spark.sql.Row> getAllRows(String[] fullPartitionPaths) {
- return HoodieClientTestUtils
- .read(jsc, basePath, sqlContext, fs, fullPartitionPaths);
+ return HoodieClientTestUtils.read(jsc, basePath, sqlContext, storage, fullPartitionPaths);
}
private String getFullPartitionPath(String relativePartitionPath) {
@@ -1208,7 +1215,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
String file1 = statuses.get(0).getFileId();
assertEquals(100,
- fileUtils.readRowKeys(hadoopConf, new Path(basePath, statuses.get(0).getStat().getPath()))
+ fileUtils.readRowKeys(hadoopConf, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
.size(), "file should contain 100 records");
// Update + Inserts such that they just expand file1
@@ -1227,7 +1234,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(1, statuses.size(), "Just 1 file needs to be updated.");
assertEquals(file1, statuses.get(0).getFileId(), "Existing file should be expanded");
assertEquals(commitTime1, statuses.get(0).getStat().getPrevCommit(), "Existing file should be expanded");
- Path newFile = new Path(basePath, statuses.get(0).getStat().getPath());
+ StoragePath newFile = new StoragePath(basePath, statuses.get(0).getStat().getPath());
assertEquals(140, fileUtils.readRowKeys(hadoopConf, newFile).size(),
"file should contain 140 records");
@@ -1262,7 +1269,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
for (HoodieBaseFile file : files) {
if (file.getFileName().contains(file1)) {
assertEquals(commitTime3, file.getCommitTime(), "Existing file should be expanded");
- records = fileUtils.readAvroRecords(hadoopConf, new Path(file.getPath()));
+ records = fileUtils.readAvroRecords(hadoopConf, new StoragePath(file.getPath()));
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
String recordCommitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
@@ -1278,7 +1285,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(0, keys2.size(), "All keys added in commit 2 must be updated in commit3 correctly");
} else {
assertEquals(commitTime3, file.getCommitTime(), "New file must be written for commit 3");
- records = fileUtils.readAvroRecords(hadoopConf, new Path(file.getPath()));
+ records = fileUtils.readAvroRecords(hadoopConf, new StoragePath(file.getPath()));
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
assertEquals(commitTime3, record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString(),
@@ -1315,11 +1322,11 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 1);
List<WriteStatus> statuses = client.insert(insertRecordsRDD1, commitTime1).collect();
assertNoWriteErrors(statuses);
- assertPartitionMetadata(basePath, new String[] {testPartitionPath}, fs);
+ assertPartitionMetadata(basePath, new String[] {testPartitionPath}, storage);
assertEquals(1, statuses.size(), "Just 1 file needs to be added.");
String file1 = statuses.get(0).getFileId();
assertEquals(100,
- fileUtils.readRowKeys(hadoopConf, new Path(basePath, statuses.get(0).getStat().getPath()))
+ fileUtils.readRowKeys(hadoopConf, new StoragePath(basePath, statuses.get(0).getStat().getPath()))
.size(), "file should contain 100 records");
// Second, set of Inserts should just expand file1
@@ -1334,7 +1341,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertEquals(file1, statuses.get(0).getFileId(), "Existing file should be expanded");
assertEquals(commitTime1, statuses.get(0).getStat().getPrevCommit(), "Existing file should be expanded");
- Path newFile = new Path(basePath, statuses.get(0).getStat().getPath());
+ StoragePath newFile = new StoragePath(basePath, statuses.get(0).getStat().getPath());
assertEquals(140, fileUtils.readRowKeys(hadoopConf, newFile).size(),
"file should contain 140 records");
List<GenericRecord> records = fileUtils.readAvroRecords(hadoopConf, newFile);
@@ -1356,8 +1363,8 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertNoWriteErrors(statuses);
assertEquals(2, statuses.size(), "2 files needs to be committed.");
assertEquals(340,
- fileUtils.readRowKeys(hadoopConf, new Path(basePath, statuses.get(0).getStat().getPath())).size()
- + fileUtils.readRowKeys(hadoopConf, new Path(basePath, statuses.get(1).getStat().getPath())).size(),
+ fileUtils.readRowKeys(hadoopConf, new StoragePath(basePath, statuses.get(0).getStat().getPath())).size()
+ + fileUtils.readRowKeys(hadoopConf, new StoragePath(basePath, statuses.get(1).getStat().getPath())).size(),
"file should contain 340 records");
HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
@@ -1369,7 +1376,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
int totalInserts = 0;
for (HoodieBaseFile file : files) {
assertEquals(commitTime3, file.getCommitTime(), "All files must be at commit 3");
- totalInserts += fileUtils.readAvroRecords(hadoopConf, new Path(file.getPath())).size();
... 32669 lines suppressed ...