You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by si...@apache.org on 2025/02/12 15:17:21 UTC
(hudi) branch master updated: [HUDI-5612] Add support for using metadata table with spillable map based file system views (#12110)
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new eb20c2ce676 [HUDI-5612] Add support for using metadata table with spillable map based file system views (#12110)
eb20c2ce676 is described below
commit eb20c2ce676a85c6fef9c613b9b56121f910535c
Author: Tim Brown <ti...@onehouse.ai>
AuthorDate: Wed Feb 12 09:17:14 2025 -0600
[HUDI-5612] Add support for using metadata table with spillable map based file system views (#12110)
---
.../cli/commands/TestFileSystemViewCommand.java | 4 +-
.../apache/hudi/client/CompactionAdminClient.java | 11 +-
.../hudi/client/utils/MetadataTableUtils.java | 45 ----
.../java/org/apache/hudi/io/HoodieCDCLogger.java | 2 +
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 2 +
.../metadata/HoodieBackedTableMetadataWriter.java | 17 +-
.../java/org/apache/hudi/table/HoodieTable.java | 4 +-
.../action/clean/CleanPlanActionExecutor.java | 5 +-
.../hudi/table/action/clean/CleanPlanner.java | 2 +-
.../action/savepoint/SavepointActionExecutor.java | 3 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 8 +-
.../testutils/HoodieJavaClientTestHarness.java | 4 +-
.../client/utils/SparkMetadataWriterUtils.java | 4 +-
.../hudi/client/TestCompactionAdminClient.java | 10 +-
.../functional/TestExternalPathHandling.java | 5 +-
.../functional/TestHoodieFileSystemViews.java | 275 +++++++++++++++++++++
.../table/action/compact/CompactionTestBase.java | 2 +-
.../TestTimelineServerBasedWriteMarkers.java | 3 +-
.../hudi/testutils/HoodieClientTestUtils.java | 2 +-
.../testutils/HoodieSparkClientTestHarness.java | 2 +-
.../apache/hudi/common/model/HoodieFileGroup.java | 20 ++
.../common/serialization/CustomSerializer.java | 20 +-
.../common/serialization/DefaultSerializer.java | 21 +-
.../serialization/HoodieFileGroupSerializer.java | 88 +++++++
.../table/log/HoodieMergedLogRecordScanner.java | 3 +-
.../read/HoodieBaseFileGroupRecordBuffer.java | 3 +-
.../table/view/AbstractTableFileSystemView.java | 104 ++------
.../common/table/view/FileSystemViewManager.java | 97 +++++---
.../table/view/HoodieFileGroupSizeEstimator.java | 44 ++++
.../table/view/HoodieTableFileSystemView.java | 28 ++-
.../IncrementalTimelineSyncFileSystemView.java | 5 +-
.../hudi/common/table/view/NoOpTableMetadata.java | 148 +++++++++++
.../table/view/PriorityBasedFileSystemView.java | 14 +-
.../table/view/RocksDbBasedFileSystemView.java | 35 +--
.../view/SpillableMapBasedFileSystemView.java | 107 ++++----
.../common/util/collection/BitCaskDiskMap.java | 29 ++-
.../hudi/common/util/collection/DiskMap.java | 2 +-
.../util/collection/ExternalSpillableMap.java | 22 +-
.../common/util/collection/LazyFileIterable.java | 17 +-
.../hudi/common/util/collection/RocksDBDAO.java | 53 ++--
.../common/util/collection/RocksDbDiskMap.java | 12 +-
.../metadata/FileSystemBackedTableMetadata.java | 21 ++
.../hudi/metadata/HoodieBackedTableMetadata.java | 53 ++--
.../metadata/HoodieMetadataFileSystemView.java | 123 ---------
.../hudi/metadata/HoodieMetadataMetrics.java | 8 +-
.../apache/hudi/metadata/HoodieTableMetadata.java | 7 +
.../hudi/metadata/HoodieTableMetadataUtil.java | 10 +-
.../SecondaryIndexRecordGenerationUtils.java | 3 +-
.../serialization/TestDefaultSerializer.java | 52 ++++
.../TestHoodieFileGroupSerializer.java | 82 ++++++
.../hudi/common/table/log/TestLogFileIterator.java | 2 +
.../table/read/TestHoodieFileGroupReaderBase.java | 7 +-
.../view/TestHoodieFileGroupSizeEstimator.java | 62 +++++
.../view/TestPriorityBasedFileSystemView.java | 103 ++++----
.../org/apache/hudi/table/format/FormatUtils.java | 2 +
.../apache/hudi/sink/compact/TestCustomSerDe.java | 19 +-
.../TestRocksDBBasedIncrementalFSViewSync.java | 6 +-
.../table/view/TestRocksDbBasedFileSystemView.java | 6 +-
.../view/TestSpillableMapBasedFileSystemView.java | 6 +-
...TestSpillableMapBasedIncrementalFSViewSync.java | 6 +-
.../common/testutils/HoodieCommonTestHarness.java | 13 +-
.../common/util/collection/TestBitCaskDiskMap.java | 213 ++++++++--------
.../util/collection/TestExternalSpillableMap.java | 19 +-
.../util/collection/TestRocksDbBasedMap.java | 28 ++-
.../common/util/collection/TestRocksDbDiskMap.java | 225 +++++++++--------
.../realtime/HoodieMergeOnReadSnapshotReader.java | 2 +
.../reader/DFSHoodieDatasetInputReader.java | 4 +-
.../connect/writers/BufferedConnectWriter.java | 2 +
.../table/view/HoodieMetaserverFileSystemView.java | 5 +-
.../org/apache/hudi/cdc/CDCFileGroupIterator.scala | 7 +-
.../ShowColumnStatsOverlapProcedure.scala | 2 +-
.../ShowMetadataTableColumnStatsProcedure.scala | 9 +-
.../hudi/TestHoodieMergeHandleWithSparkMerger.java | 2 +-
.../functional/TestHoodieBackedMetadata.java | 11 +-
.../TestMetadataUtilRLIandSIRecordGeneration.java | 23 +-
.../TestRemoteFileSystemViewWithMetadataTable.java | 5 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 4 +-
.../functional/TestBloomFiltersIndexSupport.scala | 8 +-
.../functional/TestColumnStatsIndexWithSQL.scala | 14 +-
.../TestHoodieMultipleBaseFileFormat.scala | 2 +-
.../hudi/functional/TestPartitionStatsIndex.scala | 7 +-
.../TestPartitionStatsIndexWithSql.scala | 18 +-
.../functional/TestSecondaryIndexPruning.scala | 9 +-
.../functional/TestSixToFiveDowngradeHandler.scala | 5 +-
.../hudi/command/index/TestExpressionIndex.scala | 18 +-
.../hudi/dml/TestPartialUpdateForMergeInto.scala | 2 +-
.../hudi/timeline/service/TimelineService.java | 6 +-
.../TestRemoteHoodieTableFileSystemView.java | 3 +-
.../utilities/HoodieMetadataTableValidator.java | 22 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 4 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 8 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 6 +-
.../TestHoodieMetadataTableValidator.java | 8 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 10 +-
94 files changed, 1674 insertions(+), 910 deletions(-)
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java
index c9424dc7cbd..9d786704ccb 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java
@@ -116,7 +116,7 @@ public class TestFileSystemViewCommand extends CLIFunctionalTestHarness {
// Reload meta client and create fsView
metaClient = HoodieTableMetaClient.reload(metaClient);
- nonpartitionedFsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline(), true);
+ nonpartitionedFsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context(), metaClient, metaClient.getActiveTimeline(), true);
}
private void createPartitionedTable() throws IOException {
@@ -159,7 +159,7 @@ public class TestFileSystemViewCommand extends CLIFunctionalTestHarness {
// Reload meta client and create fsView
metaClient = HoodieTableMetaClient.reload(metaClient);
- partitionedFsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline(), true);
+ partitionedFsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context(), metaClient, metaClient.getActiveTimeline(), true);
}
/**
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
index b7736c6b2f2..3b3bd2dc6ee 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
@@ -79,8 +79,7 @@ public class CompactionAdminClient extends BaseHoodieClient {
public List<ValidationOpResult> validateCompactionPlan(HoodieTableMetaClient metaClient, String compactionInstant,
int parallelism) throws IOException {
HoodieCompactionPlan plan = getCompactionPlan(metaClient, compactionInstant);
- HoodieTableFileSystemView fsView =
- new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline());
+ HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(getEngineContext(), metaClient, metaClient.getCommitsAndCompactionTimeline());
if (plan.getOperations() != null) {
List<CompactionOperation> ops = plan.getOperations().stream()
@@ -88,7 +87,7 @@ public class CompactionAdminClient extends BaseHoodieClient {
context.setJobStatus(this.getClass().getSimpleName(), "Validate compaction operations: " + config.getTableName());
return context.map(ops, op -> {
try {
- return validateCompactionOperation(metaClient, compactionInstant, op, Option.of(fsView));
+ return validateCompactionOperation(metaClient, compactionInstant, op, fsView);
} catch (IOException e) {
throw new HoodieIOException(e.getMessage(), e);
}
@@ -225,12 +224,10 @@ public class CompactionAdminClient extends BaseHoodieClient {
* @param metaClient Hoodie Table Meta client
* @param compactionInstant Compaction Instant
* @param operation Compaction Operation
- * @param fsViewOpt File System View
+ * @param fileSystemView File System View
*/
private ValidationOpResult validateCompactionOperation(HoodieTableMetaClient metaClient, String compactionInstant,
- CompactionOperation operation, Option<HoodieTableFileSystemView> fsViewOpt) throws IOException {
- HoodieTableFileSystemView fileSystemView = fsViewOpt.isPresent() ? fsViewOpt.get()
- : new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline());
+ CompactionOperation operation,HoodieTableFileSystemView fileSystemView) throws IOException {
Option<HoodieInstant> lastInstant = metaClient.getCommitsAndCompactionTimeline().lastInstant();
try {
if (lastInstant.isPresent()) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataTableUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataTableUtils.java
deleted file mode 100644
index 5e4ed6ac3d9..00000000000
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/MetadataTableUtils.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hudi.client.utils;
-
-import org.apache.hudi.common.table.HoodieTableConfig;
-import org.apache.hudi.common.table.view.FileSystemViewStorageType;
-import org.apache.hudi.config.HoodieWriteConfig;
-
-public class MetadataTableUtils {
-
- /**
- * Whether to use batch lookup for listing the latest base files in metadata table.
- * <p>
- * Note that metadata table has to be enabled, and the storage type of the file system view
- * cannot be EMBEDDED_KV_STORE or SPILLABLE_DISK (these two types are not integrated with
- * metadata table, see HUDI-5612).
- *
- * @param config Write configs.
- * @return {@code true} if using batch lookup; {@code false} otherwise.
- */
- public static boolean shouldUseBatchLookup(HoodieTableConfig tableConfig, HoodieWriteConfig config) {
- FileSystemViewStorageType storageType =
- config.getClientSpecifiedViewStorageConfig().getStorageType();
- return tableConfig.isMetadataTableAvailable()
- && !FileSystemViewStorageType.EMBEDDED_KV_STORE.equals(storageType)
- && !FileSystemViewStorageType.SPILLABLE_DISK.equals(storageType);
- }
-}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
index 5677f4188bd..7d5cac60d4b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
@@ -22,6 +22,7 @@ import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
import org.apache.hudi.common.model.HoodieAvroPayload;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.cdc.HoodieCDCOperation;
import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode;
@@ -138,6 +139,7 @@ public class HoodieCDCLogger implements Closeable {
new DefaultSizeEstimator<>(),
new DefaultSizeEstimator<>(),
config.getCommonConfig().getSpillableDiskMapType(),
+ new DefaultSerializer<>(),
config.getCommonConfig().isBitCaskDiskMapCompressionEnabled());
this.transformer = getTransformer();
this.maxBlockSize = config.getLogFileDataBlockMaxSize();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
index a966ff0666f..e1b139a7472 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
@@ -32,6 +32,7 @@ import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats;
import org.apache.hudi.common.model.IOType;
import org.apache.hudi.common.model.MetadataValues;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.util.DefaultSizeEstimator;
import org.apache.hudi.common.util.HoodieRecordSizeEstimator;
import org.apache.hudi.common.util.Option;
@@ -245,6 +246,7 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
this.keyToNewRecords = new ExternalSpillableMap<>(memoryForMerge, config.getSpillableMapBasePath(),
new DefaultSizeEstimator(), new HoodieRecordSizeEstimator(writeSchema),
config.getCommonConfig().getSpillableDiskMapType(),
+ new DefaultSerializer<>(),
config.getCommonConfig().isBitCaskDiskMapCompressionEnabled());
} catch (IOException io) {
throw new HoodieIOException("Cannot instantiate an ExternalSpillableMap", io);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 4d3622e231a..a7fea3bbe8a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -158,7 +158,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
protected final List<MetadataPartitionType> enabledPartitionTypes;
// Is the MDT bootstrapped and ready to be read from
private boolean initialized = false;
- private HoodieMetadataFileSystemView metadataView;
+ private HoodieTableFileSystemView metadataView;
/**
* Hudi backed table metadata writer.
@@ -215,11 +215,11 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
}
}
- private HoodieMetadataFileSystemView getMetadataView() {
+ private HoodieTableFileSystemView getMetadataView() {
if (metadataView == null || !metadataView.equals(metadata.getMetadataFileSystemView())) {
ValidationUtils.checkState(metadata != null, "Metadata table not initialized");
ValidationUtils.checkState(dataMetaClient != null, "Data table meta client not initialized");
- metadataView = new HoodieMetadataFileSystemView(dataMetaClient, dataMetaClient.getActiveTimeline(), metadata);
+ metadataView = new HoodieTableFileSystemView(metadata, dataMetaClient, dataMetaClient.getActiveTimeline());
}
return metadataView;
}
@@ -649,7 +649,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
private List<Pair<String, FileSlice>> getPartitionFileSlicePairs() throws IOException {
String latestInstant = dataMetaClient.getActiveTimeline().filterCompletedAndCompactionInstants().lastInstant()
.map(HoodieInstant::requestedTime).orElse(SOLO_COMMIT_TIMESTAMP);
- try (HoodieMetadataFileSystemView fsView = getMetadataView()) {
+ try (HoodieTableFileSystemView fsView = getMetadataView()) {
// Collect the list of latest file slices present in each partition
List<String> partitions = metadata.getAllPartitionPaths();
fsView.loadAllPartitions();
@@ -661,7 +661,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
}
private Pair<Integer, HoodieData<HoodieRecord>> initializeRecordIndexPartition() throws IOException {
- final HoodieMetadataFileSystemView fsView = getMetadataView();
+ final HoodieTableFileSystemView fsView = getMetadataView();
final HoodieTable hoodieTable = getTable(dataWriteConfig, dataMetaClient);
// Collect the list of latest base files present in each partition
@@ -1174,7 +1174,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
HoodieIndexDefinition indexDefinition = getIndexDefinition(indexPartition);
// Load file system view for only the affected partitions on the driver.
// By loading on the driver one time, we avoid loading the same metadata multiple times on the executors.
- HoodieMetadataFileSystemView fsView = getMetadataView();
+ HoodieTableFileSystemView fsView = getMetadataView();
fsView.loadPartitions(new ArrayList<>(commitMetadata.getWritePartitionPaths()));
return convertWriteStatsToSecondaryIndexRecords(allWriteStats, instantTime, indexDefinition, dataWriteConfig.getMetadataConfig(), fsView, dataMetaClient, engineContext, getEngineType());
}
@@ -1323,6 +1323,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
writeClient = null;
}
if (metadataView != null) {
+ metadataView.close();
metadataView = null;
}
}
@@ -1427,7 +1428,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
protected HoodieData<HoodieRecord> prepRecords(Map<String, HoodieData<HoodieRecord>> partitionRecordsMap) {
// The result set
HoodieData<HoodieRecord> allPartitionRecords = engineContext.emptyHoodieData();
- try (HoodieTableFileSystemView fsView = HoodieTableMetadataUtil.getFileSystemView(metadataMetaClient)) {
+ try (HoodieTableFileSystemView fsView = HoodieTableMetadataUtil.getFileSystemViewForMetadataTable(metadataMetaClient)) {
for (Map.Entry<String, HoodieData<HoodieRecord>> entry : partitionRecordsMap.entrySet()) {
final String partitionName = entry.getKey();
HoodieData<HoodieRecord> records = entry.getValue();
@@ -1641,7 +1642,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
}
private HoodieData<HoodieRecord> getRecordIndexReplacedRecords(HoodieReplaceCommitMetadata replaceCommitMetadata) {
- HoodieMetadataFileSystemView fsView = getMetadataView();
+ HoodieTableFileSystemView fsView = getMetadataView();
List<Pair<String, HoodieBaseFile>> partitionBaseFilePairs = replaceCommitMetadata
.getPartitionToReplaceFileIds()
.keySet().stream()
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
index cfd37994806..3539b3404c8 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -169,7 +169,7 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
private synchronized FileSystemViewManager getViewManager() {
if (null == viewManager) {
- viewManager = FileSystemViewManager.createViewManager(getContext(), config.getViewStorageConfig(), config.getCommonConfig(), unused -> getMetadataTable());
+ viewManager = FileSystemViewManager.createViewManager(getContext(), config.getMetadataConfig(), config.getViewStorageConfig(), config.getCommonConfig(), unused -> getMetadataTable());
}
return viewManager;
}
@@ -340,7 +340,7 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
* Get the view of the file system for this table.
*/
public TableFileSystemView getFileSystemView() {
- return new HoodieTableFileSystemView(metaClient, getCompletedCommitsTimeline());
+ return HoodieTableFileSystemView.fileListingBasedFileSystemView(getContext(), metaClient, getCompletedCommitsTimeline());
}
/**
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java
index 074a87cc635..4616ec6883c 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java
@@ -48,9 +48,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import static org.apache.hudi.client.utils.MetadataTableUtils.shouldUseBatchLookup;
-import static org.apache.hudi.common.util.MapUtils.nonEmpty;
import static org.apache.hudi.common.util.CleanerUtils.SAVEPOINTED_TIMESTAMPS;
+import static org.apache.hudi.common.util.MapUtils.nonEmpty;
public class CleanPlanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K, O, Option<HoodieCleanerPlan>> {
@@ -129,7 +128,7 @@ public class CleanPlanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I
Map<String, List<HoodieCleanFileInfo>> cleanOps = new HashMap<>();
List<String> partitionsToDelete = new ArrayList<>();
- boolean shouldUseBatchLookup = shouldUseBatchLookup(table.getMetaClient().getTableConfig(), config);
+ boolean shouldUseBatchLookup = table.getMetaClient().getTableConfig().isMetadataTableAvailable();
for (int i = 0; i < partitionsToClean.size(); i += cleanerParallelism) {
// Handles at most 'cleanerParallelism' number of partitions once at a time to avoid overlarge memory pressure to the timeline server
// (remote or local embedded), thus to reduce the risk of an OOM exception.
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
index 893dfe8548a..a5682ecb6a9 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
@@ -463,7 +463,7 @@ public class CleanPlanner<T, I, K, O> implements Serializable {
*/
private boolean hasPendingFiles(String partitionPath) {
try {
- HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(hoodieTable.getMetaClient(), hoodieTable.getActiveTimeline());
+ HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context, hoodieTable.getMetaClient(), hoodieTable.getActiveTimeline());
StoragePath fullPartitionPath = new StoragePath(hoodieTable.getMetaClient().getBasePath(), partitionPath);
fsView.addFilesToView(partitionPath, FSUtils.getAllDataFilesInPartition(
hoodieTable.getStorage(), fullPartitionPath));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
index 30ec3faf541..fd809f43ca2 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
@@ -43,7 +43,6 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import static org.apache.hudi.client.utils.MetadataTableUtils.shouldUseBatchLookup;
import static org.apache.hudi.common.table.timeline.HoodieInstant.State.REQUESTED;
import static org.apache.hudi.common.table.timeline.InstantComparison.GREATER_THAN_OR_EQUALS;
import static org.apache.hudi.common.table.timeline.InstantComparison.compareTimestamps;
@@ -110,7 +109,7 @@ public class SavepointActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I
// (2) using direct file system listing: we parallelize the partition listing so that
// each partition can be listed on the file system concurrently through Spark.
// Note that
- if (shouldUseBatchLookup(table.getMetaClient().getTableConfig(), config)) {
+ if (table.getMetaClient().getTableConfig().isMetadataTableAvailable()) {
latestFilesMap = view.getAllLatestFileSlicesBeforeOrOn(instantTime).entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey,
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
index 9992441649e..cfee5a022e9 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
@@ -91,7 +91,6 @@ import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
import org.apache.hudi.metadata.HoodieBackedTableMetadata;
import org.apache.hudi.metadata.HoodieBackedTableMetadataWriter;
-import org.apache.hudi.metadata.HoodieMetadataFileSystemView;
import org.apache.hudi.metadata.HoodieMetadataLogRecordReader;
import org.apache.hudi.metadata.HoodieMetadataMetrics;
import org.apache.hudi.metadata.HoodieMetadataPayload;
@@ -2591,9 +2590,8 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTableMetadata tableMetadata = metadata(client);
- HoodieMetadataFileSystemView metadataFileSystemView = new HoodieMetadataFileSystemView(
- metaClient, metaClient.reloadActiveTimeline(), tableMetadata);
- HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline());
+ HoodieTableFileSystemView metadataFileSystemView = new HoodieTableFileSystemView(tableMetadata, metaClient, metaClient.reloadActiveTimeline());
+ HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context, metaClient, metaClient.getActiveTimeline());
tableMetadata.getAllPartitionPaths().forEach(partition -> {
List<String> fileNamesFromMetadataFileListing = metadataFileSystemView.getLatestBaseFiles(partition)
.map(baseFile -> baseFile.getFileName())
@@ -2832,7 +2830,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// Metadata table should automatically compact and clean
// versions are +1 as autoclean / compaction happens end of commits
int numFileVersions = metadataWriteConfig.getCleanerFileVersionsRetained() + 1;
- HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metadataMetaClient, metadataMetaClient.getActiveTimeline());
+ HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(engineContext, metadataMetaClient, metadataMetaClient.getActiveTimeline());
metadataTablePartitions.forEach(partition -> {
List<FileSlice> latestSlices = fsView.getLatestFileSlices(partition).collect(Collectors.toList());
assertTrue(latestSlices.stream().map(FileSlice::getBaseFile).count() <= latestSlices.size(), "Should have a single latest base file per file group");
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
index 650eb12326d..cc5432ae92b 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
@@ -431,7 +431,9 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
// Metadata table should automatically compact and clean
// versions are +1 as autoClean / compaction happens end of commits
int numFileVersions = metadataWriteConfig.getCleanerFileVersionsRetained() + 1;
- HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metadataMetaClient, metadataMetaClient.getActiveTimeline());
+ HoodieTableMetadata fileBasedTableMetadata = new FileSystemBackedTableMetadata(engineContext, metadataMetaClient.getTableConfig(), metadataMetaClient.getStorage(),
+ metadataMetaClient.getBasePath().toString());
+ HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(fileBasedTableMetadata, metadataMetaClient, metadataMetaClient.getActiveTimeline());
metadataTablePartitions.forEach(partition -> {
List<FileSlice> latestSlices = fsView.getLatestFileSlices(partition).collect(Collectors.toList());
assertTrue(latestSlices.stream().map(FileSlice::getBaseFile).filter(Objects::nonNull).count() > 0, "Should have a single latest base file");
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java
index 15069004f56..fb324261e5d 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java
@@ -108,7 +108,7 @@ import static org.apache.hudi.metadata.HoodieMetadataPayload.createBloomFilterMe
import static org.apache.hudi.metadata.HoodieMetadataPayload.createColumnStatsRecords;
import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_BLOOM_FILTERS;
import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS;
-import static org.apache.hudi.metadata.HoodieTableMetadataUtil.getFileSystemView;
+import static org.apache.hudi.metadata.HoodieTableMetadataUtil.getFileSystemViewForMetadataTable;
import static org.apache.hudi.metadata.MetadataPartitionType.COLUMN_STATS;
/**
@@ -403,7 +403,7 @@ public class SparkMetadataWriterUtils {
// Step 2: Compute expression index records for the modified partitions
LOG.debug("Indexing following columns for partition stats index: {}", validColumnsToIndex);
List<String> partitionPaths = new ArrayList<>(commitMetadata.getWritePartitionPaths());
- HoodieTableFileSystemView fileSystemView = getFileSystemView(dataMetaClient);
+ HoodieTableFileSystemView fileSystemView = getFileSystemViewForMetadataTable(dataMetaClient);
int parallelism = Math.max(Math.min(partitionPaths.size(), metadataConfig.getPartitionStatsIndexParallelism()), 1);
return engineContext.parallelize(partitionPaths, parallelism).mapToPair(partitionName -> {
checkState(tableMetadata != null, "tableMetadata should not be null when scanning metadata table");
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java
index 3806904076c..6ff32d477eb 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java
@@ -190,8 +190,7 @@ public class TestCompactionAdminClient extends HoodieClientTestBase {
// Log files belonging to file-slices created because of compaction request should not be renamed
// because the file slicing is based on completion time.
- final HoodieTableFileSystemView fsView =
- new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline());
+ final HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(getEngineContext(), metaClient, metaClient.getCommitsAndCompactionTimeline());
Map<String, Long> fileIdToCountsBeforeRenaming =
fsView.getLatestMergedFileSlicesBeforeOrOn(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0], compactionInstant)
@@ -204,8 +203,7 @@ public class TestCompactionAdminClient extends HoodieClientTestBase {
client.unscheduleCompactionPlan(compactionInstant, false, 1, false);
metaClient.reloadActiveTimeline();
- final HoodieTableFileSystemView newFsView =
- new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline());
+ final HoodieTableFileSystemView newFsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(getEngineContext(), metaClient, metaClient.getCommitsAndCompactionTimeline());
Set<String> commitsWithDataFile = CollectionUtils.createSet("000", "004");
// Expect each file-slice whose base-commit is same as compaction commit to contain no new Log files
newFsView.getLatestFileSlicesBeforeOrOn(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0], compactionInstant, true)
@@ -245,7 +243,7 @@ public class TestCompactionAdminClient extends HoodieClientTestBase {
// Log files belonging to file-slices created because of compaction request must be renamed
final HoodieTableFileSystemView fsView =
- new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline());
+ HoodieTableFileSystemView.fileListingBasedFileSystemView(getEngineContext(), metaClient, metaClient.getCommitsAndCompactionTimeline());
Map<String, Long> fileIdToCountsBeforeRenaming =
fsView.getLatestMergedFileSlicesBeforeOrOn(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0], compactionInstant)
@@ -259,7 +257,7 @@ public class TestCompactionAdminClient extends HoodieClientTestBase {
metaClient = HoodieTestUtils.createMetaClient(metaClient.getStorageConf(), basePath);
final HoodieTableFileSystemView newFsView =
- new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline());
+ HoodieTableFileSystemView.fileListingBasedFileSystemView(getEngineContext(), metaClient, metaClient.getCommitsAndCompactionTimeline());
// Expect all file-slice whose base-commit is same as compaction commit to contain no new Log files
newFsView.getLatestFileSlicesBeforeOrOn(HoodieTestUtils.DEFAULT_PARTITION_PATHS[0], compactionInstant, true)
.filter(fs -> fs.getBaseInstantTime().equals(compactionInstant))
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
index 10b75b8e859..3b7de3b2740 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestExternalPathHandling.java
@@ -47,7 +47,7 @@ import org.apache.hudi.config.HoodieArchivalConfig;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metadata.HoodieBackedTableMetadata;
-import org.apache.hudi.metadata.HoodieMetadataFileSystemView;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.table.action.clean.CleanPlanner;
import org.apache.hudi.testutils.HoodieClientTestBase;
@@ -219,7 +219,8 @@ public class TestExternalPathHandling extends HoodieClientTestBase {
}
private void assertFileGroupCorrectness(String instantTime, String partitionPath, String filePath, String fileId, int expectedSize) {
- HoodieTableFileSystemView fsView = new HoodieMetadataFileSystemView(context, metaClient, metaClient.reloadActiveTimeline(), writeConfig.getMetadataConfig());
+ HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(context, metaClient.getStorage(), writeConfig.getMetadataConfig(), metaClient.getBasePath().toString());
+ HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(tableMetadata, metaClient, metaClient.reloadActiveTimeline());
List<HoodieFileGroup> fileGroups = fsView.getAllFileGroups(partitionPath).collect(Collectors.toList());
Assertions.assertEquals(expectedSize, fileGroups.size());
Option<HoodieFileGroup> fileGroupOption = Option.fromJavaOptional(fileGroups.stream().filter(fg -> fg.getFileGroupId().getFileId().equals(fileId)).findFirst());
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieFileSystemViews.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieFileSystemViews.java
new file mode 100644
index 00000000000..ed63cbabbe9
--- /dev/null
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieFileSystemViews.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.client.functional;
+
+import org.apache.hudi.client.SparkRDDWriteClient;
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.function.SerializableFunctionUnchecked;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieBaseFile;
+import org.apache.hudi.common.model.HoodieLogFile;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.model.WriteOperationType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.view.FileSystemViewManager;
+import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
+import org.apache.hudi.common.table.view.FileSystemViewStorageType;
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieArchivalConfig;
+import org.apache.hudi.config.HoodieCleanConfig;
+import org.apache.hudi.config.HoodieClusteringConfig;
+import org.apache.hudi.config.HoodieCompactionConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.testutils.HoodieClientTestBase;
+
+import org.apache.spark.api.java.JavaRDD;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
+import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Tests diff file system views.
+ */
+public class TestHoodieFileSystemViews extends HoodieClientTestBase {
+
+ private HoodieTableType tableType = HoodieTableType.COPY_ON_WRITE;
+
+ protected HoodieTableType getTableType() {
+ return tableType;
+ }
+
+ public static List<Arguments> tableTypeMetadataFSVTypeArgs() {
+ List<Arguments> testCases = new ArrayList<>();
+ for (HoodieTableType tableType : HoodieTableType.values()) {
+ for (boolean enableMdt : Arrays.asList(true, false)) {
+ for (FileSystemViewStorageType viewStorageType : Arrays.asList(FileSystemViewStorageType.MEMORY, FileSystemViewStorageType.SPILLABLE_DISK)) {
+ for (int writerVersion : Arrays.asList(6, 8)) {
+ testCases.add(Arguments.of(tableType, enableMdt, viewStorageType, writerVersion));
+ }
+ }
+ }
+ }
+ return testCases;
+ }
+
+ @ParameterizedTest
+ @MethodSource("tableTypeMetadataFSVTypeArgs")
+ public void testFileSystemViewConsistency(HoodieTableType tableType, boolean enableMdt, FileSystemViewStorageType storageType, int writeVersion) throws IOException {
+ this.tableType = tableType;
+ HoodieWriteConfig.Builder configBuilder = getConfigBuilder();
+ if (tableType == HoodieTableType.MERGE_ON_READ) {
+ configBuilder.withCompactionConfig(HoodieCompactionConfig.newBuilder().withInlineCompaction(true)
+ .withMaxNumDeltaCommitsBeforeCompaction(3).build());
+ }
+ configBuilder
+ .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder().withStorageType(storageType).build())
+ .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(enableMdt).build())
+ .withClusteringConfig(HoodieClusteringConfig.newBuilder().withInlineClustering(true).withInlineClusteringNumCommits(5).build())
+ .withCleanConfig(HoodieCleanConfig.newBuilder().retainCommits(4).build())
+ // set aggressive values so that within 20 batches few iterations of cleaner and archival will kick in
+ .withArchivalConfig(HoodieArchivalConfig.newBuilder().archiveCommitsWith(6, 8).build())
+ .withWriteTableVersion(writeVersion);
+ HoodieWriteConfig config = configBuilder.build();
+ try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
+ insertRecords(client, "001", 100, WriteOperationType.BULK_INSERT);
+ insertRecords(client, "002", 100, WriteOperationType.INSERT);
+ assertFileSystemViews(config, enableMdt, storageType);
+ for (int i = 3; i < 10; i++) {
+ String commitTime = String.format("%10d", i);
+ upsertRecords(client, commitTime, 50);
+ }
+ assertFileSystemViews(config, enableMdt, storageType);
+ for (int i = 10; i < 20; i++) {
+ String commitTime = String.format("%10d", i);
+ upsertRecords(client, commitTime, 50);
+ }
+
+ // mimic failed write for last completed operation and retry few more operations.
+ HoodieInstant lastInstant = metaClient.reloadActiveTimeline().lastInstant().get();
+ StoragePath instantPath = HoodieTestUtils
+ .getCompleteInstantPath(metaClient.getStorage(),
+ metaClient.getTimelinePath(),
+ lastInstant.requestedTime(), lastInstant.getAction());
+ metaClient.getStorage().deleteFile(instantPath);
+
+ assertFileSystemViews(config, enableMdt, storageType);
+ // add few more updates
+ for (int i = 21; i < 23; i++) {
+ String commitTime = String.format("%10d", i);
+ upsertRecords(client, commitTime, 50);
+ }
+ }
+ assertFileSystemViews(config, enableMdt, storageType);
+ }
+
+ private void assertFileSystemViews(HoodieWriteConfig writeConfig, boolean enableMdt, FileSystemViewStorageType baseStorageType) {
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+ // base line file system view is in-memory for any combination.
+ HoodieTableFileSystemView expectedFileSystemView = FileSystemViewManager.createInMemoryFileSystemView(context, metaClient,
+ HoodieMetadataConfig.newBuilder().enable(false).build());
+
+ // to be compared against.
+ // if no mdt enabled, compare w/ spillable.
+ // if mdt is enabled, depending on storage type, either it will be mdt fsv or spillable fsv w/ mdt enabled.
+ FileSystemViewStorageConfig viewStorageConfig = FileSystemViewStorageConfig.newBuilder().fromProperties(writeConfig.getProps())
+ .withStorageType(baseStorageType).build();
+ HoodieTableFileSystemView actualFileSystemView = (HoodieTableFileSystemView) FileSystemViewManager
+ .createViewManager(context, writeConfig.getMetadataConfig(), viewStorageConfig, writeConfig.getCommonConfig(),
+ (SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata>) v1 ->
+ HoodieTableMetadata.create(context, metaClient.getStorage(), writeConfig.getMetadataConfig(), writeConfig.getBasePath()))
+ .getFileSystemView(basePath);
+ try {
+ assertForFSVEquality(expectedFileSystemView, actualFileSystemView, enableMdt);
+ } finally {
+ expectedFileSystemView.close();
+ actualFileSystemView.close();
+ }
+ }
+
+ private void assertForFSVEquality(HoodieTableFileSystemView fsv1, HoodieTableFileSystemView fsv2, boolean enableMdt) {
+ List<String> allPartitionNames = Arrays.asList(DEFAULT_FIRST_PARTITION_PATH, DEFAULT_SECOND_PARTITION_PATH, DEFAULT_THIRD_PARTITION_PATH);
+ fsv1.loadPartitions(allPartitionNames);
+ if (enableMdt) {
+ // assumes 2nd one is metadata enabled. loadAllPartitions() cannot be invoked for non-mdt fsv.
+ fsv2.loadAllPartitions();
+ } else {
+ fsv2.loadPartitions(allPartitionNames);
+ }
+ List<StoragePath> allPartitions1 = fsv1.getPartitionPaths();
+ List<StoragePath> allPartitions2 = fsv2.getPartitionPaths();
+ Collections.sort(allPartitions1);
+ Collections.sort(allPartitions2);
+ assertEquals(allPartitions1, allPartitions2);
+
+ allPartitionNames.forEach(path -> {
+ List<HoodieBaseFile> latestBaseFiles1 = fsv1.getLatestBaseFiles(path).collect(Collectors.toList());
+ List<HoodieBaseFile> latestBaseFiles2 = fsv2.getLatestBaseFiles(path).collect(Collectors.toList());
+ assertBaseFileListEquality(latestBaseFiles1, latestBaseFiles2);
+
+ List<FileSlice> fileSlices1 = fsv1.getLatestFileSlices(path).collect(Collectors.toList());
+ List<FileSlice> fileSlices2 = fsv2.getLatestFileSlices(path).collect(Collectors.toList());
+ assertFileSliceListEquality(fileSlices1, fileSlices2);
+ });
+ }
+
+ private void assertBaseFileListEquality(List<HoodieBaseFile> baseFileList1, List<HoodieBaseFile> baseFileList2) {
+ assertEquals(baseFileList1.size(), baseFileList2.size());
+ Map<String, HoodieBaseFile> fileNameToBaseFileMap1 = new HashMap<>();
+ baseFileList1.forEach(entry -> {
+ fileNameToBaseFileMap1.put(entry.getFileName(), entry);
+ });
+ Map<String, HoodieBaseFile> fileNameToBaseFileMap2 = new HashMap<>();
+ baseFileList2.forEach(entry -> {
+ fileNameToBaseFileMap2.put(entry.getFileName(), entry);
+ });
+ fileNameToBaseFileMap1.entrySet().forEach((kv) -> {
+ assertTrue(fileNameToBaseFileMap2.containsKey(kv.getKey()));
+ assertBaseFileEquality(kv.getValue(), fileNameToBaseFileMap2.get(kv.getKey()));
+ });
+ }
+
+ public void assertBaseFileEquality(HoodieBaseFile baseFile1, HoodieBaseFile baseFile2) {
+ assertEquals(baseFile1.getFileName(), baseFile2.getFileName());
+ assertEquals(baseFile1.getFileId(), baseFile2.getFileId());
+ assertEquals(baseFile1.getFileLen(), baseFile2.getFileLen());
+ assertEquals(baseFile1.getFileSize(), baseFile2.getFileSize());
+ }
+
+ private void assertFileSliceListEquality(List<FileSlice> fileSlices1, List<FileSlice> fileSlices2) {
+ assertEquals(fileSlices1.size(), fileSlices1.size());
+ Map<Pair<String, String>, FileSlice> fileNameToFileSliceMap1 = new HashMap<>();
+ fileSlices1.forEach(entry -> {
+ fileNameToFileSliceMap1.put(Pair.of(entry.getFileId(), entry.getBaseInstantTime()), entry);
+ });
+ Map<Pair<String, String>, FileSlice> fileNameToFileSliceMap2 = new HashMap<>();
+ fileSlices2.forEach(entry -> {
+ fileNameToFileSliceMap2.put(Pair.of(entry.getFileId(), entry.getBaseInstantTime()), entry);
+ });
+ fileNameToFileSliceMap1.entrySet().forEach((kv) -> {
+ assertTrue(fileNameToFileSliceMap2.containsKey(kv.getKey()));
+ assertFileSliceEquality(kv.getValue(), fileNameToFileSliceMap2.get(kv.getKey()));
+ });
+ }
+
+ private void assertFileSliceEquality(FileSlice fileSlice1, FileSlice fileSlice2) {
+ assertEquals(fileSlice1.getBaseFile().isPresent(), fileSlice2.getBaseFile().isPresent());
+ if (fileSlice1.getBaseFile().isPresent()) {
+ assertBaseFileEquality(fileSlice1.getBaseFile().get(), fileSlice2.getBaseFile().get());
+ }
+ List<HoodieLogFile> logFiles1 = fileSlice1.getLogFiles().collect(Collectors.toList());
+ List<HoodieLogFile> logFiles2 = fileSlice2.getLogFiles().collect(Collectors.toList());
+ assertEquals(logFiles1.size(), logFiles2.size());
+ int counter = 0;
+ for (HoodieLogFile logFile1 : logFiles1) {
+ HoodieLogFile logFile2 = logFiles2.get(counter++);
+ assertLogFileEquality(logFile1, logFile2);
+ }
+ }
+
+ private void assertLogFileEquality(HoodieLogFile logFile1, HoodieLogFile logFile2) {
+ assertEquals(logFile1.getFileName(), logFile2.getFileName());
+ assertEquals(logFile1.getFileId(), logFile2.getFileId());
+ assertEquals(logFile1.getLogVersion(), logFile2.getLogVersion());
+ assertEquals(logFile1.getFileSize(), logFile2.getFileSize());
+ assertEquals(logFile1.getDeltaCommitTime(), logFile2.getDeltaCommitTime());
+ assertEquals(logFile1.getFileExtension(), logFile2.getFileExtension());
+ assertEquals(logFile1.getLogWriteToken(), logFile2.getLogWriteToken());
+ }
+
+ private void insertRecords(SparkRDDWriteClient client, String commitTime, int numRecords, WriteOperationType operationType) {
+ client.startCommitWithTime(commitTime);
+ List<HoodieRecord> inserts1 = dataGen.generateInserts(commitTime, numRecords);
+ JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 2);
+ List<WriteStatus> statuses = operationType == WriteOperationType.BULK_INSERT ? client.bulkInsert(insertRecordsRDD1, commitTime, Option.empty()).collect() :
+ client.insert(insertRecordsRDD1, commitTime).collect();
+ assertNoWriteErrors(statuses);
+ }
+
+ private void upsertRecords(SparkRDDWriteClient client, String commitTime, int numRecords) {
+ client.startCommitWithTime(commitTime);
+ List<HoodieRecord> updates = dataGen.generateUniqueUpdates(commitTime, numRecords);
+ JavaRDD<HoodieRecord> updatesRdd = jsc.parallelize(updates, 2);
+ List<WriteStatus> statuses = client.upsert(updatesRdd, commitTime).collect();
+ assertNoWriteErrors(statuses);
+ }
+}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java
index b9275f223e1..5205aa72a49 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java
@@ -291,7 +291,7 @@ public class CompactionTestBase extends HoodieClientTestBase {
}
protected List<FileSlice> getCurrentLatestFileSlices(HoodieTable table) {
- HoodieTableFileSystemView view = new HoodieTableFileSystemView(table.getMetaClient(),
+ HoodieTableFileSystemView view = HoodieTableFileSystemView.fileListingBasedFileSystemView(getEngineContext(), table.getMetaClient(),
table.getMetaClient().reloadActiveTimeline().getWriteTimeline());
return Arrays.stream(HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS)
.flatMap(view::getLatestFileSlices).collect(Collectors.toList());
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java
index 6f3d9b67ed6..aae8d294bc3 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java
@@ -20,6 +20,7 @@ package org.apache.hudi.table.marker;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.config.HoodieCommonConfig;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.table.marker.MarkerType;
import org.apache.hudi.common.table.view.FileSystemViewManager;
@@ -69,7 +70,7 @@ public class TestTimelineServerBasedWriteMarkers extends TestWriteMarkersBase {
timelineService = new TimelineService(localEngineContext, HadoopFSUtils.getStorageConf(),
TimelineService.Config.builder().serverPort(0).enableMarkerRequests(true).build(),
storage,
- FileSystemViewManager.createViewManager(localEngineContext, storageConf, HoodieCommonConfig.newBuilder().build()));
+ FileSystemViewManager.createViewManager(localEngineContext, HoodieMetadataConfig.newBuilder().build(), storageConf, HoodieCommonConfig.newBuilder().build()));
timelineService.startService();
} catch (Exception ex) {
throw new RuntimeException(ex);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java
index c305ccd42f0..9a0abcb7922 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java
@@ -289,7 +289,7 @@ public class HoodieClientTestUtils {
TimelineService.Config.builder().enableMarkerRequests(true)
.serverPort(config.getViewStorageConfig().getRemoteViewServerPort()).build(),
HoodieStorageUtils.getStorage(HoodieTestUtils.getDefaultStorageConf()),
- FileSystemViewManager.createViewManager(context, config.getViewStorageConfig(), config.getCommonConfig()));
+ FileSystemViewManager.createViewManager(context, config.getMetadataConfig(), config.getViewStorageConfig(), config.getCommonConfig()));
timelineService.startService();
LOG.info("Timeline service server port: " + timelineServicePort);
return timelineService;
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
index ce43e86fc8a..a4dee7afe2a 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
@@ -656,7 +656,7 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
// Metadata table should automatically compact and clean
// versions are +1 as autoClean / compaction happens end of commits
int numFileVersions = metadataWriteConfig.getCleanerFileVersionsRetained() + 1;
- HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metadataMetaClient, metadataMetaClient.getActiveTimeline());
+ HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(engineContext, metadataMetaClient, metadataMetaClient.getActiveTimeline());
metadataTablePartitions.forEach(partition -> {
List<FileSlice> latestSlices = fsView.getLatestFileSlices(partition).collect(Collectors.toList());
assertTrue(latestSlices.stream().map(FileSlice::getBaseFile).filter(Objects::nonNull).count() > 0, "Should have a single latest base file");
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java
index 1fcc003165b..d8b14780d74 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java
@@ -27,6 +27,7 @@ import org.apache.hudi.common.util.VisibleForTesting;
import java.io.Serializable;
import java.util.Comparator;
import java.util.List;
+import java.util.Objects;
import java.util.TreeMap;
import java.util.stream.Stream;
@@ -254,4 +255,23 @@ public class HoodieFileGroup implements Serializable {
public HoodieTimeline getTimeline() {
return timeline;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ HoodieFileGroup fileGroup = (HoodieFileGroup) o;
+ return Objects.equals(fileGroupId, fileGroup.fileGroupId)
+ && Objects.equals(fileSlices, fileGroup.fileSlices)
+ && Objects.equals(lastInstant, fileGroup.lastInstant);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(fileGroupId, fileSlices, lastInstant);
+ }
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/serialization/CustomSerializer.java
similarity index 56%
copy from hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java
copy to hudi-common/src/main/java/org/apache/hudi/common/serialization/CustomSerializer.java
index 7793178d102..316210fdc6c 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/serialization/CustomSerializer.java
@@ -16,22 +16,18 @@
* limitations under the License.
*/
-package org.apache.hudi.common.table.view;
-
-import org.apache.hudi.common.table.timeline.HoodieTimeline;
+package org.apache.hudi.common.serialization;
import java.io.IOException;
-import java.nio.file.Files;
+import java.io.Serializable;
/**
- * Tests RocksDB based file system view {@link RocksDbBasedFileSystemView}.
+ * Allows custom serialization for spillable map values.
+ * @param <T>
*/
-public class TestRocksDbBasedFileSystemView extends TestHoodieTableFileSystemView {
+public interface CustomSerializer<T> extends Serializable {
+
+ byte[] serialize(T input) throws IOException;
- @Override
- protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException {
- String subdirPath = Files.createTempDirectory(tempDir, null).toAbsolutePath().toString();
- return new RocksDbBasedFileSystemView(metaClient, timeline,
- FileSystemViewStorageConfig.newBuilder().withRocksDBPath(subdirPath).build());
- }
+ T deserialize(byte[] bytes);
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/serialization/DefaultSerializer.java
similarity index 57%
copy from hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java
copy to hudi-common/src/main/java/org/apache/hudi/common/serialization/DefaultSerializer.java
index 7793178d102..c45e018b549 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/serialization/DefaultSerializer.java
@@ -16,22 +16,25 @@
* limitations under the License.
*/
-package org.apache.hudi.common.table.view;
+package org.apache.hudi.common.serialization;
-import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.util.SerializationUtils;
import java.io.IOException;
-import java.nio.file.Files;
+import java.io.Serializable;
/**
- * Tests RocksDB based file system view {@link RocksDbBasedFileSystemView}.
+ * A default implementation of the interface that works for any serializable object.
+ * @param <T> class must implement {@link Serializable}
*/
-public class TestRocksDbBasedFileSystemView extends TestHoodieTableFileSystemView {
+public class DefaultSerializer<T extends Serializable> implements CustomSerializer<T> {
+ @Override
+ public byte[] serialize(T input) throws IOException {
+ return SerializationUtils.serialize(input);
+ }
@Override
- protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException {
- String subdirPath = Files.createTempDirectory(tempDir, null).toAbsolutePath().toString();
- return new RocksDbBasedFileSystemView(metaClient, timeline,
- FileSystemViewStorageConfig.newBuilder().withRocksDBPath(subdirPath).build());
+ public T deserialize(byte[] bytes) {
+ return SerializationUtils.deserialize(bytes);
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/serialization/HoodieFileGroupSerializer.java b/hudi-common/src/main/java/org/apache/hudi/common/serialization/HoodieFileGroupSerializer.java
new file mode 100644
index 00000000000..477df1f5acf
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/common/serialization/HoodieFileGroupSerializer.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.serialization;
+
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieFileGroup;
+import org.apache.hudi.common.model.HoodieFileGroupId;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.util.SerializationUtils;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Serializes {@link HoodieFileGroup} but keeps the timeline in memory since file groups
+ * for a single view will share the same {@link HoodieTimeline} instance.
+ * Deserialization will create a new instance so recreating timeline objects can lead to unnecessary bloat.
+ * IMPORTANT: This serializer can only be used in cases where the data is kept on the local machine and cannot be used
+ * when serializing between spark driver and executor.
+ */
+public class HoodieFileGroupSerializer implements CustomSerializer<List<HoodieFileGroup>> {
+ private final Map<HoodieFileGroupId, HoodieTimeline> fileGroupIdHoodieTimelineMap;
+
+ public HoodieFileGroupSerializer() {
+ this.fileGroupIdHoodieTimelineMap = new HashMap<>();
+ }
+
+ @Override
+ public byte[] serialize(List<HoodieFileGroup> input) throws IOException {
+ List<HoodieFileGroupLite> fileGroupLites = input.stream().map(fileGroup -> {
+ if (fileGroup.getTimeline() == null) {
+ throw new IllegalStateException("Timeline is null for FileGroup: '" + fileGroup.getFileGroupId().toString() + "'. All filegroup states: ["
+ + input.stream().map(fg -> fg.getFileGroupId().toString() + ":" + ((fg.getTimeline() == null) ? "NULL" : "OK")).collect(Collectors.joining(",")) + "]");
+ }
+ fileGroupIdHoodieTimelineMap.put(fileGroup.getFileGroupId(), fileGroup.getTimeline());
+ return new HoodieFileGroupLite(fileGroup.getAllRawFileSlices().collect(Collectors.toList()), fileGroup.getFileGroupId());
+ }).collect(Collectors.toList());
+ return SerializationUtils.serialize(fileGroupLites);
+ }
+
+ @Override
+ public List<HoodieFileGroup> deserialize(byte[] bytes) {
+ List<HoodieFileGroupLite> fileGroupLites = SerializationUtils.deserialize(bytes);
+ return fileGroupLites.stream().map(fileGroupLite -> {
+ HoodieTimeline timeline = fileGroupIdHoodieTimelineMap.get(fileGroupLite.fileGroupId);
+ if (timeline == null) {
+ throw new IllegalStateException("Timeline for fileGroupId: '" + fileGroupLite.fileGroupId + "' was not found in the map. Available fileGroupId in map: ["
+ + fileGroupIdHoodieTimelineMap.keySet().stream().map(HoodieFileGroupId::toString).collect(Collectors.joining(",")) + "]. FileGroupId that we are deserializing: ["
+ + fileGroupLites.stream().map(fg -> fg.fileGroupId.toString()).collect(Collectors.joining(",")) + "]");
+ }
+ HoodieFileGroup fileGroup = new HoodieFileGroup(fileGroupLite.fileGroupId, timeline);
+ fileGroupLite.fileSlices.forEach(fileGroup::addFileSlice);
+ return fileGroup;
+ }).collect(Collectors.toList());
+ }
+
+ private static class HoodieFileGroupLite implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final List<FileSlice> fileSlices;
+ private final HoodieFileGroupId fileGroupId;
+
+ HoodieFileGroupLite(List<FileSlice> fileSlices, HoodieFileGroupId fileGroupId) {
+ this.fileSlices = fileSlices;
+ this.fileGroupId = fileGroupId;
+ }
+ }
+}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java
index fb1d621eb4a..4f5d0ac4639 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java
@@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodiePreCombineAvroRecordMerger;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordMerger;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.DefaultSizeEstimator;
@@ -110,7 +111,7 @@ public class HoodieMergedLogRecordScanner extends AbstractHoodieLogRecordScanner
this.maxMemorySizeInBytes = maxMemorySizeInBytes;
// Store merged records for all versions for this log file, set the in-memory footprint to maxInMemoryMapSize
this.records = new ExternalSpillableMap<>(maxMemorySizeInBytes, spillableMapBasePath, new DefaultSizeEstimator(),
- new HoodieRecordSizeEstimator(readerSchema), diskMapType, isBitCaskDiskMapCompressionEnabled);
+ new HoodieRecordSizeEstimator(readerSchema), diskMapType, new DefaultSerializer<>(), isBitCaskDiskMapCompressionEnabled);
this.scannedPrefixes = new HashSet<>();
} catch (IOException e) {
throw new HoodieIOException("IOException when creating ExternalSpillableMap at " + spillableMapBasePath, e);
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieBaseFileGroupRecordBuffer.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieBaseFileGroupRecordBuffer.java
index 8c2a2555491..a9e1e290edb 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieBaseFileGroupRecordBuffer.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieBaseFileGroupRecordBuffer.java
@@ -30,6 +30,7 @@ import org.apache.hudi.common.model.HoodieOperation;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordMerger;
import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.log.KeySpec;
import org.apache.hudi.common.table.log.block.HoodieDataBlock;
@@ -137,7 +138,7 @@ public abstract class HoodieBaseFileGroupRecordBuffer<T> implements HoodieFileGr
try {
// Store merged records for all versions for this log file, set the in-memory footprint to maxInMemoryMapSize
this.records = new ExternalSpillableMap<>(maxMemorySizeInBytes, spillableMapBasePath, new DefaultSizeEstimator<>(),
- new HoodieRecordSizeEstimator<>(readerSchema), diskMapType, isBitCaskDiskMapCompressionEnabled);
+ new HoodieRecordSizeEstimator<>(readerSchema), diskMapType, new DefaultSerializer<>(), isBitCaskDiskMapCompressionEnabled);
} catch (IOException e) {
throw new HoodieIOException("IOException when creating ExternalSpillableMap at " + spillableMapBasePath, e);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
index bcc32e01064..df5f8241dc0 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
@@ -44,6 +44,7 @@ import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -57,7 +58,6 @@ import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -94,6 +94,7 @@ import static org.apache.hudi.common.table.timeline.InstantComparison.compareTim
public abstract class AbstractTableFileSystemView implements SyncableFileSystemView, Serializable {
private static final Logger LOG = LoggerFactory.getLogger(AbstractTableFileSystemView.class);
+ protected final HoodieTableMetadata tableMetadata;
protected HoodieTableMetaClient metaClient;
@@ -114,6 +115,10 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
private BootstrapIndex bootstrapIndex;
+ protected AbstractTableFileSystemView(HoodieTableMetadata tableMetadata) {
+ this.tableMetadata = tableMetadata;
+ }
+
/**
* Initialize the view.
*/
@@ -182,7 +187,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
if (!isPartitionAvailableInStore(partition)) {
if (bootstrapIndex.useIndex()) {
try (BootstrapIndex.IndexReader reader = bootstrapIndex.createReader()) {
- LOG.info("Bootstrap Index available for partition " + partition);
+ LOG.info("Bootstrap Index available for partition {}", partition);
List<BootstrapFileMapping> sourceFileMappings =
reader.getSourceFileMappingForPartition(partition);
addBootstrapBaseFileMapping(sourceFileMappings.stream()
@@ -194,9 +199,8 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
}
});
long storePartitionsTs = timer.endTimer();
- LOG.debug("addFilesToView: NumFiles=" + statuses.size() + ", NumFileGroups=" + fileGroups.size()
- + ", FileGroupsCreationTime=" + fgBuildTimeTakenMs
- + ", StoreTimeTaken=" + storePartitionsTs);
+ LOG.debug("addFilesToView: NumFiles={}, NumFileGroups={}, FileGroupsCreationTime={}, StoreTimeTaken={}",
+ statuses.size(), fileGroups.size(), fgBuildTimeTakenMs, storePartitionsTs);
return fileGroups;
}
@@ -289,8 +293,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
Map<HoodieFileGroupId, HoodieInstant> replacedFileGroups = resultStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
(instance1, instance2) -> compareTimestamps(instance1.requestedTime(), LESSER_THAN, instance2.requestedTime()) ? instance2 : instance1));
resetReplacedFileGroups(replacedFileGroups);
- LOG.info("Took " + hoodieTimer.endTimer() + " ms to read " + replacedTimeline.countInstants() + " instants, "
- + replacedFileGroups.size() + " replaced file groups");
+ LOG.info("Took {} ms to read {} instants, {} replaced file groups", hoodieTimer.endTimer(), replacedTimeline.countInstants(), replacedFileGroups.size());
}
@Override
@@ -300,7 +303,10 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
this.metaClient = null;
this.completionTimeQueryView = null;
this.visibleCommitsAndCompactionTimeline = null;
+ tableMetadata.close();
clear();
+ } catch (Exception ex) {
+ throw new HoodieException("Unable to close file system view", ex);
} finally {
writeLock.unlock();
}
@@ -318,6 +324,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
clear();
// Initialize with new Hoodie timeline.
init(metaClient, getTimeline());
+ tableMetadata.reset();
} finally {
writeLock.unlock();
}
@@ -345,7 +352,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
private List<String> ensureAllPartitionsLoadedCorrectly() {
ValidationUtils.checkArgument(!isClosed(), "View is already closed");
try {
- List<String> formattedPartitionList = getAllPartitionPaths().stream()
+ List<String> formattedPartitionList = tableMetadata.getAllPartitionPaths().stream()
.map(this::formatPartitionKey).collect(Collectors.toList());
ensurePartitionsLoadedCorrectly(formattedPartitionList);
return formattedPartitionList;
@@ -375,7 +382,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
long beginTs = System.currentTimeMillis();
// Not loaded yet
try {
- LOG.debug("Building file system view for partitions: " + partitionSet);
+ LOG.debug("Building file system view for partitions: {}", partitionSet);
// Pairs of relative partition path and absolute partition path
List<Pair<String, StoragePath>> absolutePartitionPathList = partitionSet.stream()
@@ -384,22 +391,22 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
.collect(Collectors.toList());
long beginLsTs = System.currentTimeMillis();
Map<Pair<String, StoragePath>, List<StoragePathInfo>> pathInfoMap =
- listPartitions(absolutePartitionPathList);
+ tableMetadata.listPartitions(absolutePartitionPathList);
long endLsTs = System.currentTimeMillis();
- LOG.debug("Time taken to list partitions " + partitionSet + " =" + (endLsTs - beginLsTs));
+ LOG.debug("Time taken to list partitions {} ={}", partitionSet, (endLsTs - beginLsTs));
pathInfoMap.forEach((partitionPair, statuses) -> {
String relativePartitionStr = partitionPair.getLeft();
List<HoodieFileGroup> groups = addFilesToView(relativePartitionStr, statuses);
if (groups.isEmpty()) {
storePartitionView(relativePartitionStr, Collections.emptyList());
}
- LOG.debug("#files found in partition (" + relativePartitionStr + ") =" + statuses.size());
+ LOG.debug("#files found in partition ({}) ={}", relativePartitionStr, statuses.size());
});
} catch (IOException e) {
throw new HoodieIOException("Failed to list base files in partitions " + partitionSet, e);
}
long endTs = System.currentTimeMillis();
- LOG.debug("Time to load partition " + partitionSet + " =" + (endTs - beginTs));
+ LOG.debug("Time to load partition {} ={}", partitionSet, (endTs - beginTs));
}
partitionSet.forEach(partition ->
@@ -408,43 +415,6 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
}
}
- /**
- * @return A list of relative partition paths of all partitions.
- * @throws IOException upon error.
- */
- protected List<String> getAllPartitionPaths() throws IOException {
- throw new HoodieException("Getting all partition paths with file system listing sequentially "
- + "can be very slow. This should not be invoked.");
- }
-
- /**
- * @param partitionPathList A list of pairs of the relative and absolute paths of the partitions.
- * @return all the files from the partitions.
- * @throws IOException upon error.
- */
- protected Map<Pair<String, StoragePath>, List<StoragePathInfo>> listPartitions(
- List<Pair<String, StoragePath>> partitionPathList) throws IOException {
- Map<Pair<String, StoragePath>, List<StoragePathInfo>> pathInfoMap = new HashMap<>();
-
- for (Pair<String, StoragePath> partitionPair : partitionPathList) {
- StoragePath absolutePartitionPath = partitionPair.getRight();
- try {
- pathInfoMap.put(partitionPair,
- metaClient.getStorage().listDirectEntries(absolutePartitionPath));
- } catch (IOException e) {
- if (!metaClient.getStorage().exists(absolutePartitionPath)) {
- pathInfoMap.put(partitionPair, Collections.emptyList());
- } else {
- // in case the partition path was created by another caller
- pathInfoMap.put(partitionPair,
- metaClient.getStorage().listDirectEntries(absolutePartitionPath));
- }
- }
- }
-
- return pathInfoMap;
- }
-
/**
* Returns all files situated at the given partition.
*/
@@ -453,11 +423,10 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
StoragePath partitionPath = FSUtils.constructAbsolutePath(metaClient.getBasePath(),
relativePartitionPath);
long beginLsTs = System.currentTimeMillis();
- List<StoragePathInfo> pathInfoList = listPartition(partitionPath);
+ List<StoragePathInfo> pathInfoList = tableMetadata.getAllFilesInPartition(partitionPath);
long endLsTs = System.currentTimeMillis();
LOG.debug(
- "#files found in partition (" + relativePartitionPath + ") =" + pathInfoList.size()
- + ", " + "Time taken =" + (endLsTs - beginLsTs));
+ "#files found in partition ({}}) = {}, Time taken ={}", relativePartitionPath, pathInfoList.size(), (endLsTs - beginLsTs));
return pathInfoList;
}
@@ -476,7 +445,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
if (!isPartitionAvailableInStore(partitionPathStr)) {
// Not loaded yet
try {
- LOG.info("Building file system view for partition (" + partitionPathStr + ")");
+ LOG.info("Building file system view for partition ({})", partitionPathStr);
List<HoodieFileGroup> groups = addFilesToView(partitionPathStr, getAllFilesInPartition(partitionPathStr));
if (groups.isEmpty()) {
storePartitionView(partitionPathStr, new ArrayList<>());
@@ -485,33 +454,14 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
throw new HoodieIOException("Failed to list base files in partition " + partitionPathStr, e);
}
} else {
- LOG.debug("View already built for Partition :" + partitionPathStr + ", FOUND is ");
+ LOG.debug("View already built for Partition :{}, FOUND is ", partitionPathStr);
}
long endTs = System.currentTimeMillis();
- LOG.debug("Time to load partition (" + partitionPathStr + ") =" + (endTs - beginTs));
+ LOG.debug("Time to load partition ({}) ={}", partitionPathStr, (endTs - beginTs));
return true;
});
}
- /**
- * Return all the files from the partition.
- *
- * @param partitionPath The absolute path of the partition
- * @throws IOException
- */
- protected List<StoragePathInfo> listPartition(StoragePath partitionPath) throws IOException {
- try {
- return metaClient.getStorage().listDirectEntries(partitionPath);
- } catch (IOException e) {
- if (!metaClient.getStorage().exists(partitionPath)) {
- return Collections.emptyList();
- } else {
- // in case the partition path was created by another caller
- return metaClient.getStorage().listDirectEntries(partitionPath);
- }
- }
- }
-
/**
* Helper to convert file-status to base-files.
*
@@ -641,7 +591,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
private FileSlice filterUncommittedLogs(FileSlice fileSlice) {
List<HoodieLogFile> committedLogFiles = fileSlice.getLogFiles().filter(logFile -> completionTimeQueryView.isCompleted(logFile.getDeltaCommitTime())).collect(Collectors.toList());
if (committedLogFiles.size() != fileSlice.getLogFiles().count()) {
- LOG.debug("File Slice (" + fileSlice + ") has uncommitted log files.");
+ LOG.debug("File Slice ({}) has uncommitted log files.", fileSlice);
// A file is filtered out of the file-slice if the corresponding
// instant has not completed yet.
FileSlice transformed = new FileSlice(fileSlice.getPartitionPath(), fileSlice.getBaseInstantTime(), fileSlice.getFileId());
@@ -1192,7 +1142,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
private Map<HoodieFileGroupId, BootstrapBaseFileMapping> getBootstrapBaseFileMappings(String partition) {
try (BootstrapIndex.IndexReader reader = bootstrapIndex.createReader()) {
- LOG.info("Bootstrap Index available for partition " + partition);
+ LOG.info("Bootstrap Index available for partition {}", partition);
List<BootstrapFileMapping> sourceFileMappings =
reader.getSourceFileMappingForPartition(partition);
return sourceFileMappings.stream()
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
index 66d63323076..0c7495fa294 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
@@ -23,13 +23,12 @@ import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.config.HoodieMetaserverConfig;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.function.SerializableFunctionUnchecked;
-import org.apache.hudi.common.function.SerializableSupplier;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Functions.Function2;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.ValidationUtils;
-import org.apache.hudi.metadata.HoodieMetadataFileSystemView;
+import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.storage.StorageConfiguration;
@@ -134,14 +133,19 @@ public class FileSystemViewManager {
/**
* Create RocksDB based file System view for a table.
*
- * @param viewConf View Storage Configuration
+ * @param engineContext engine context used
+ * @param viewConf View Storage Configuration
* @param metaClient HoodieTableMetaClient
- * @return {@link RocksDbBasedFileSystemView}
+ * @param metadataCreator creates {@link HoodieTableMetadata}
+ * @return
*/
- private static RocksDbBasedFileSystemView createRocksDBBasedFileSystemView(FileSystemViewStorageConfig viewConf,
- HoodieTableMetaClient metaClient) {
+ private static RocksDbBasedFileSystemView createRocksDBBasedFileSystemView(HoodieEngineContext engineContext, FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient,
+ boolean metadataTableEnabled,
+ SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator) {
+ LOG.info("Creating RocksDB based view for basePath {}.", metaClient.getBasePath());
HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
- return new RocksDbBasedFileSystemView(metaClient, timeline, viewConf);
+ HoodieTableMetadata tableMetadata = getTableMetadata(engineContext, metaClient, metadataTableEnabled, metadataCreator);
+ return new RocksDbBasedFileSystemView(tableMetadata, metaClient, timeline, viewConf);
}
/**
@@ -151,32 +155,42 @@ public class FileSystemViewManager {
* @param metaClient HoodieTableMetaClient
* @return {@link SpillableMapBasedFileSystemView}
*/
- private static SpillableMapBasedFileSystemView createSpillableMapBasedFileSystemView(
- FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient, HoodieCommonConfig commonConfig) {
+ private static SpillableMapBasedFileSystemView createSpillableMapBasedFileSystemView(HoodieEngineContext engineContext, FileSystemViewStorageConfig viewConf,
+ HoodieTableMetaClient metaClient, HoodieCommonConfig commonConfig,
+ boolean metadataTableEnabled,
+ SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator) {
LOG.info("Creating SpillableMap based view for basePath {}.", metaClient.getBasePath());
HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
- return new SpillableMapBasedFileSystemView(metaClient, timeline, viewConf, commonConfig);
+ HoodieTableMetadata tableMetadata = getTableMetadata(engineContext, metaClient, metadataTableEnabled, metadataCreator);
+ return new SpillableMapBasedFileSystemView(tableMetadata, metaClient, timeline, viewConf, commonConfig);
}
/**
* Create an in-memory file System view for a table.
*/
- private static HoodieTableFileSystemView createInMemoryFileSystemView(
- FileSystemViewStorageConfig viewConf,
- HoodieTableMetaClient metaClient,
- SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator) {
+ private static HoodieTableFileSystemView createInMemoryFileSystemView(HoodieEngineContext engineContext, FileSystemViewStorageConfig viewConf,
+ HoodieTableMetaClient metaClient, boolean metadataTableEnabled,
+ SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator) {
LOG.info("Creating InMemory based view for basePath {}.", metaClient.getBasePath());
HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
- if (metaClient.getTableConfig().isMetadataTableAvailable()) {
- ValidationUtils.checkArgument(metadataCreator != null, "Metadata supplier is null. Cannot instantiate metadata file system view");
- return new HoodieMetadataFileSystemView(metaClient, timeline, metadataCreator.apply(metaClient));
- }
+ HoodieTableMetadata tableMetadata = getTableMetadata(engineContext, metaClient, metadataTableEnabled, metadataCreator);
if (metaClient.getMetaserverConfig().isMetaserverEnabled()) {
return (HoodieTableFileSystemView) ReflectionUtils.loadClass(HOODIE_METASERVER_FILE_SYSTEM_VIEW_CLASS,
new Class<?>[]{HoodieTableMetaClient.class, HoodieTimeline.class, HoodieMetaserverConfig.class},
metaClient, timeline, metaClient.getMetaserverConfig());
}
- return new HoodieTableFileSystemView(metaClient, timeline, viewConf.isIncrementalTimelineSyncEnabled());
+ return new HoodieTableFileSystemView(tableMetadata, metaClient, timeline, viewConf.isIncrementalTimelineSyncEnabled());
+ }
+
+ private static HoodieTableMetadata getTableMetadata(HoodieEngineContext engineContext, HoodieTableMetaClient metaClient,
+ boolean metadataTableEnabled,
+ SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator) {
+ if (metadataTableEnabled && metaClient.getTableConfig().isMetadataTableAvailable()) {
+ ValidationUtils.checkArgument(metadataCreator != null, "Metadata supplier is null. Cannot instantiate metadata file system view");
+ return metadataCreator.apply(metaClient);
+ } else {
+ return new FileSystemBackedTableMetadata(engineContext, metaClient.getTableConfig(), metaClient.getStorage(), metaClient.getBasePath().toString());
+ }
}
public static HoodieTableFileSystemView createInMemoryFileSystemView(
@@ -191,15 +205,15 @@ public class FileSystemViewManager {
HoodieMetadataConfig metadataConfig,
HoodieTimeline timeline) {
LOG.info("Creating InMemory based view for basePath {}.", metaClient.getBasePath());
- if (metaClient.getTableConfig().isMetadataTableAvailable()) {
- return new HoodieMetadataFileSystemView(engineContext, metaClient, timeline, metadataConfig);
- }
+ HoodieTableMetadata tableMetadata = getTableMetadata(engineContext, metaClient, metadataConfig.isEnabled(),
+ unused -> HoodieTableMetadata.create(engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePath().toString()));
+
if (metaClient.getMetaserverConfig().isMetaserverEnabled()) {
return (HoodieTableFileSystemView) ReflectionUtils.loadClass(HOODIE_METASERVER_FILE_SYSTEM_VIEW_CLASS,
new Class<?>[]{HoodieTableMetaClient.class, HoodieTimeline.class, HoodieMetaserverConfig.class},
metaClient, timeline, metaClient.getMetaserverConfig());
}
- return new HoodieTableFileSystemView(metaClient, timeline);
+ return new HoodieTableFileSystemView(tableMetadata, metaClient, timeline);
}
/**
@@ -221,37 +235,40 @@ public class FileSystemViewManager {
final HoodieMetadataConfig metadataConfig,
final FileSystemViewStorageConfig config,
final HoodieCommonConfig commonConfig) {
- return createViewManager(context, config, commonConfig,
+ return createViewManager(context, metadataConfig, config, commonConfig,
metaClient -> HoodieTableMetadata.create(context, metaClient.getStorage(), metadataConfig, metaClient.getBasePath().toString(), true));
}
public static FileSystemViewManager createViewManager(final HoodieEngineContext context,
+ final HoodieMetadataConfig metadataConfig,
final FileSystemViewStorageConfig config,
final HoodieCommonConfig commonConfig) {
- return createViewManager(context, config, commonConfig, null);
+ return createViewManager(context, metadataConfig, config, commonConfig, null);
}
/**
* Main Factory method for building file-system views.
*/
public static FileSystemViewManager createViewManager(final HoodieEngineContext context,
+ final HoodieMetadataConfig metadataConfig,
final FileSystemViewStorageConfig config,
final HoodieCommonConfig commonConfig,
final SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator) {
LOG.info("Creating View Manager with storage type {}.", config.getStorageType());
+ boolean metadataTableEnabled = metadataConfig.isEnabled();
switch (config.getStorageType()) {
case EMBEDDED_KV_STORE:
LOG.debug("Creating embedded rocks-db based Table View");
return new FileSystemViewManager(context, config,
- (metaClient, viewConf) -> createRocksDBBasedFileSystemView(viewConf, metaClient));
+ (metaClient, viewConf) -> createRocksDBBasedFileSystemView(context, viewConf, metaClient, metadataTableEnabled, metadataCreator));
case SPILLABLE_DISK:
LOG.debug("Creating Spillable Disk based Table View");
return new FileSystemViewManager(context, config,
- (metaClient, viewConf) -> createSpillableMapBasedFileSystemView(viewConf, metaClient, commonConfig));
+ (metaClient, viewConf) -> createSpillableMapBasedFileSystemView(context, viewConf, metaClient, commonConfig, metadataTableEnabled, metadataCreator));
case MEMORY:
LOG.debug("Creating in-memory based Table View");
return new FileSystemViewManager(context, config,
- (metaClient, viewConfig) -> createInMemoryFileSystemView(viewConfig, metaClient, metadataCreator));
+ (metaClient, viewConfig) -> createInMemoryFileSystemView(context, viewConfig, metaClient, metadataTableEnabled, metadataCreator));
case REMOTE_ONLY:
LOG.debug("Creating remote only table view");
return new FileSystemViewManager(context, config, (metaClient, viewConfig) -> createRemoteFileSystemView(viewConfig,
@@ -261,38 +278,40 @@ public class FileSystemViewManager {
return new FileSystemViewManager(context, config, (metaClient, viewConfig) -> {
RemoteHoodieTableFileSystemView remoteFileSystemView =
createRemoteFileSystemView(viewConfig, metaClient);
- SerializableSupplier<SyncableFileSystemView> secondaryViewSupplier = new SecondaryViewSupplier(viewConfig, metaClient, commonConfig, metadataCreator);
- return new PriorityBasedFileSystemView(remoteFileSystemView, secondaryViewSupplier);
+ SecondaryViewCreator secondaryViewSupplier = new SecondaryViewCreator(viewConfig, metaClient, commonConfig, metadataTableEnabled, metadataCreator);
+ return new PriorityBasedFileSystemView(remoteFileSystemView, secondaryViewSupplier, context);
});
default:
throw new IllegalArgumentException("Unknown file system view type :" + config.getStorageType());
}
}
- private static class SecondaryViewSupplier implements SerializableSupplier<SyncableFileSystemView> {
- private final FileSystemViewStorageConfig viewConfig;
+ static class SecondaryViewCreator implements SerializableFunctionUnchecked<HoodieEngineContext, SyncableFileSystemView> { private final FileSystemViewStorageConfig viewConfig;
private final HoodieTableMetaClient metaClient;
private final HoodieCommonConfig commonConfig;
+ private final boolean metadataTableEnabled;
private final SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator;
- private SecondaryViewSupplier(FileSystemViewStorageConfig viewConfig,
- HoodieTableMetaClient metaClient, HoodieCommonConfig commonConfig,
- SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator) {
+ SecondaryViewCreator(FileSystemViewStorageConfig viewConfig,
+ HoodieTableMetaClient metaClient, HoodieCommonConfig commonConfig,
+ boolean metadataTableEnabled,
+ SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator) {
this.viewConfig = viewConfig;
this.metaClient = metaClient;
this.commonConfig = commonConfig;
+ this.metadataTableEnabled = metadataTableEnabled;
this.metadataCreator = metadataCreator;
}
@Override
- public SyncableFileSystemView get() {
+ public SyncableFileSystemView apply(HoodieEngineContext engineContext) {
switch (viewConfig.getSecondaryStorageType()) {
case MEMORY:
- return createInMemoryFileSystemView(viewConfig, metaClient, metadataCreator);
+ return createInMemoryFileSystemView(engineContext, viewConfig, metaClient, metadataTableEnabled, metadataCreator);
case EMBEDDED_KV_STORE:
- return createRocksDBBasedFileSystemView(viewConfig, metaClient);
+ return createRocksDBBasedFileSystemView(engineContext, viewConfig, metaClient, metadataTableEnabled, metadataCreator);
case SPILLABLE_DISK:
- return createSpillableMapBasedFileSystemView(viewConfig, metaClient, commonConfig);
+ return createSpillableMapBasedFileSystemView(engineContext, viewConfig, metaClient, commonConfig, metadataTableEnabled, metadataCreator);
default:
throw new IllegalArgumentException("Secondary Storage type can only be in-memory or spillable. Was :"
+ viewConfig.getSecondaryStorageType());
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieFileGroupSizeEstimator.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieFileGroupSizeEstimator.java
new file mode 100644
index 00000000000..bf475cd96cf
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieFileGroupSizeEstimator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.table.view;
+
+import org.apache.hudi.common.model.HoodieFileGroup;
+import org.apache.hudi.common.util.ObjectSizeCalculator;
+import org.apache.hudi.common.util.SizeEstimator;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * A size estimator that is used alongside of {@link org.apache.hudi.common.serialization.HoodieFileGroupSerializer} to
+ * estimate the size of list of file groups within a {@link SpillableMapBasedFileSystemView}.
+ */
+class HoodieFileGroupSizeEstimator implements SizeEstimator<List<HoodieFileGroup>>, Serializable {
+
+ @Override
+ public long sizeEstimate(List<HoodieFileGroup> hoodieFileGroups) {
+ if (hoodieFileGroups.isEmpty()) {
+ return 0;
+ }
+ long sizeOfFileGroupIds = ObjectSizeCalculator.getObjectSize(hoodieFileGroups.get(0).getFileGroupId()) * hoodieFileGroups.size();
+ long sizeOfFileSlices = ObjectSizeCalculator.getObjectSize(hoodieFileGroups.stream().flatMap(HoodieFileGroup::getAllFileSlices).collect(Collectors.toList()));
+ return sizeOfFileSlices + sizeOfFileGroupIds;
+ }
+}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java
index 24ebadb4546..9a9bbd87304 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java
@@ -18,6 +18,7 @@
package org.apache.hudi.common.table.view;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.BootstrapBaseFileMapping;
import org.apache.hudi.common.model.CompactionOperation;
import org.apache.hudi.common.model.HoodieFileGroup;
@@ -28,6 +29,8 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.storage.StoragePathInfo;
import org.slf4j.Logger;
@@ -88,23 +91,34 @@ public class HoodieTableFileSystemView extends IncrementalTimelineSyncFileSystem
*/
private boolean closed = false;
- HoodieTableFileSystemView(boolean enableIncrementalTimelineSync) {
- super(enableIncrementalTimelineSync);
+ HoodieTableFileSystemView(HoodieTableMetadata tableMetadata, boolean enableIncrementalTimelineSync) {
+ super(tableMetadata, enableIncrementalTimelineSync);
}
/**
* Create a file system view, as of the given timeline.
*/
- public HoodieTableFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline) {
- this(metaClient, visibleActiveTimeline, false);
+ public HoodieTableFileSystemView(HoodieTableMetadata tableMetadata, HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline) {
+ this(tableMetadata, metaClient, visibleActiveTimeline, false);
+ }
+
+ public static HoodieTableFileSystemView fileListingBasedFileSystemView(HoodieEngineContext engineContext, HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline) {
+ return fileListingBasedFileSystemView(engineContext, metaClient, visibleActiveTimeline, false);
+ }
+
+ public static HoodieTableFileSystemView fileListingBasedFileSystemView(HoodieEngineContext engineContext, HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline,
+ boolean enableIncrementalSync) {
+ HoodieTableMetadata tableMetadata = new FileSystemBackedTableMetadata(engineContext, metaClient.getTableConfig(), metaClient.getStorage(),
+ metaClient.getBasePath().toString());
+ return new HoodieTableFileSystemView(tableMetadata, metaClient, visibleActiveTimeline, enableIncrementalSync);
}
/**
* Create a file system view, as of the given timeline.
*/
- public HoodieTableFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline,
+ public HoodieTableFileSystemView(HoodieTableMetadata tableMetadata, HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline,
boolean enableIncrementalTimelineSync) {
- super(enableIncrementalTimelineSync);
+ super(tableMetadata, enableIncrementalTimelineSync);
init(metaClient, visibleActiveTimeline);
}
@@ -175,7 +189,7 @@ public class HoodieTableFileSystemView extends IncrementalTimelineSyncFileSystem
*/
public HoodieTableFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline,
List<StoragePathInfo> pathInfoList) {
- this(metaClient, visibleActiveTimeline);
+ this(new NoOpTableMetadata(), metaClient, visibleActiveTimeline);
addFilesToView(pathInfoList);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java
index 078dc38ae16..7932f1f764c 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java
@@ -44,6 +44,7 @@ import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -69,7 +70,8 @@ public abstract class IncrementalTimelineSyncFileSystemView extends AbstractTabl
// This is the visible active timeline used only for incremental view syncing
private HoodieTimeline visibleActiveTimeline;
- protected IncrementalTimelineSyncFileSystemView(boolean enableIncrementalTimelineSync) {
+ protected IncrementalTimelineSyncFileSystemView(HoodieTableMetadata tableMetadata, boolean enableIncrementalTimelineSync) {
+ super(tableMetadata);
this.incrementalTimelineSyncEnabled = enableIncrementalTimelineSync;
}
@@ -84,6 +86,7 @@ public abstract class IncrementalTimelineSyncFileSystemView extends AbstractTabl
try {
writeLock.lock();
maySyncIncrementally();
+ tableMetadata.reset();
} finally {
writeLock.unlock();
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/NoOpTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/NoOpTableMetadata.java
new file mode 100644
index 00000000000..1065adf8227
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/NoOpTableMetadata.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.table.view;
+
+import org.apache.hudi.avro.model.HoodieMetadataColumnStats;
+import org.apache.hudi.common.bloom.BloomFilter;
+import org.apache.hudi.common.data.HoodieData;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieRecordGlobalLocation;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieMetadataException;
+import org.apache.hudi.expression.Expression;
+import org.apache.hudi.internal.schema.Types;
+import org.apache.hudi.metadata.HoodieMetadataPayload;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.metadata.MetadataPartitionType;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Provides an implementation of {@link HoodieTableMetadata} that cannot load any file statuses.
+ * This is used when building a {@link HoodieTableFileSystemView} based off of a set of pre-defined File Statuses so that only those files are considered within the view.
+ */
+class NoOpTableMetadata implements HoodieTableMetadata {
+ @Override
+ public List<StoragePathInfo> getAllFilesInPartition(StoragePath partitionPath) throws IOException {
+ // This should only be called when handling empty partitions since the file statuses should be pre-loaded into the view using this class
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<String> getPartitionPathWithPathPrefixUsingFilterExpression(List<String> relativePathPrefixes, Types.RecordType partitionFields, Expression expression) throws IOException {
+ throw new HoodieMetadataException("Unsupported operation: getPartitionPathWithPathPrefixUsingFilterExpression!");
+ }
+
+ @Override
+ public List<String> getPartitionPathWithPathPrefixes(List<String> relativePathPrefixes) throws IOException {
+ throw new HoodieMetadataException("Unsupported operation: getPartitionPathWithPathPrefixes!");
+ }
+
+ @Override
+ public List<String> getAllPartitionPaths() throws IOException {
+ throw new HoodieMetadataException("Unsupported operation: getAllPartitionPaths!");
+ }
+
+ @Override
+ public Map<String, List<StoragePathInfo>> getAllFilesInPartitions(Collection<String> partitionPaths) throws IOException {
+ throw new HoodieMetadataException("Unsupported operation: getAllFilesInPartitions!");
+ }
+
+ @Override
+ public Option<BloomFilter> getBloomFilter(String partitionName, String fileName) throws HoodieMetadataException {
+ throw new HoodieMetadataException("Unsupported operation: getBloomFilter!");
+ }
+
+ @Override
+ public Option<BloomFilter> getBloomFilter(String partitionName, String fileName, String metadataPartitionName) throws HoodieMetadataException {
+ throw new HoodieMetadataException("Unsupported operation: getBloomFilter!");
+ }
+
+ @Override
+ public Map<Pair<String, String>, BloomFilter> getBloomFilters(List<Pair<String, String>> partitionNameFileNameList) throws HoodieMetadataException {
+ throw new HoodieMetadataException("Unsupported operation: getBloomFilters!");
+ }
+
+ @Override
+ public Map<Pair<String, String>, BloomFilter> getBloomFilters(List<Pair<String, String>> partitionNameFileNameList, String metadataPartitionName) throws HoodieMetadataException {
+ throw new HoodieMetadataException("Unsupported operation: getBloomFilters!");
+ }
+
+ @Override
+ public Map<Pair<String, String>, HoodieMetadataColumnStats> getColumnStats(List<Pair<String, String>> partitionNameFileNameList, String columnName) throws HoodieMetadataException {
+ throw new HoodieMetadataException("Unsupported operation: getColumnsStats!");
+ }
+
+ @Override
+ public Map<String, List<HoodieRecordGlobalLocation>> readRecordIndex(List<String> recordKeys) {
+ throw new HoodieMetadataException("Unsupported operation: readRecordIndex!");
+ }
+
+ @Override
+ public Map<String, List<HoodieRecordGlobalLocation>> readSecondaryIndex(List<String> secondaryKeys, String partitionName) {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public HoodieData<HoodieRecord<HoodieMetadataPayload>> getRecordsByKeyPrefixes(List<String> keyPrefixes, String partitionName, boolean shouldLoadInMemory) {
+ throw new HoodieMetadataException("Unsupported operation: getRecordsByKeyPrefixes!");
+ }
+
+ @Override
+ public Map<String, List<HoodieRecord<HoodieMetadataPayload>>> getAllRecordsByKeys(List<String> keys, String partitionName) {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Option<String> getSyncedInstantTime() {
+ throw new HoodieMetadataException("Unsupported operation: getSyncedInstantTime!");
+ }
+
+ @Override
+ public Option<String> getLatestCompactionTime() {
+ throw new HoodieMetadataException("Unsupported operation: readRecordIndex!");
+ }
+
+ @Override
+ public void reset() {
+
+ }
+
+ @Override
+ public int getNumFileGroupsForPartition(MetadataPartitionType partition) {
+ throw new HoodieMetadataException("Unsupported operation: readRecordIndex!");
+ }
+
+ @Override
+ public Map<Pair<String, StoragePath>, List<StoragePathInfo>> listPartitions(List<Pair<String, StoragePath>> partitionPathList) throws IOException {
+ throw new HoodieMetadataException("Unsupported operation: readRecordIndex!");
+ }
+
+ @Override
+ public void close() throws Exception {
+
+ }
+}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java
index 87fb73893a7..b351553e8d5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java
@@ -18,7 +18,8 @@
package org.apache.hudi.common.table.view;
-import org.apache.hudi.common.function.SerializableSupplier;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.function.SerializableFunctionUnchecked;
import org.apache.hudi.common.model.CompactionOperation;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieBaseFile;
@@ -51,16 +52,19 @@ public class PriorityBasedFileSystemView implements SyncableFileSystemView, Seri
private static final Logger LOG = LoggerFactory.getLogger(PriorityBasedFileSystemView.class);
+ private final transient HoodieEngineContext engineContext;
private final SyncableFileSystemView preferredView;
- private final SerializableSupplier<SyncableFileSystemView> secondaryViewSupplier;
+ private final SerializableFunctionUnchecked<HoodieEngineContext, SyncableFileSystemView> secondaryViewCreator;
private SyncableFileSystemView secondaryView;
private boolean errorOnPreferredView;
- public PriorityBasedFileSystemView(SyncableFileSystemView preferredView, SerializableSupplier<SyncableFileSystemView> secondaryViewSupplier) {
+ public PriorityBasedFileSystemView(SyncableFileSystemView preferredView, SerializableFunctionUnchecked<HoodieEngineContext, SyncableFileSystemView> secondaryViewCreator,
+ HoodieEngineContext engineContext) {
this.preferredView = preferredView;
- this.secondaryViewSupplier = secondaryViewSupplier;
+ this.secondaryViewCreator = secondaryViewCreator;
this.secondaryView = null; // only initialize secondary view when required
this.errorOnPreferredView = false;
+ this.engineContext = engineContext;
}
private <R> R execute(Function0<R> preferredFunction, Function0<R> secondaryFunction) {
@@ -343,7 +347,7 @@ public class PriorityBasedFileSystemView implements SyncableFileSystemView, Seri
synchronized SyncableFileSystemView getSecondaryView() {
if (secondaryView == null) {
- secondaryView = secondaryViewSupplier.get();
+ secondaryView = secondaryViewCreator.apply(engineContext);
}
return secondaryView;
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java
index adee4ffcdc9..b6626b0659e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java
@@ -34,6 +34,7 @@ import org.apache.hudi.common.util.RocksDBSchemaHelper;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.common.util.collection.RocksDBDAO;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.storage.StoragePathInfo;
import org.slf4j.Logger;
@@ -75,18 +76,18 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
private boolean closed = false;
- public RocksDbBasedFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline,
- FileSystemViewStorageConfig config) {
- super(config.isIncrementalTimelineSyncEnabled());
+ public RocksDbBasedFileSystemView(HoodieTableMetadata tableMetadata, HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline,
+ FileSystemViewStorageConfig config) {
+ super(tableMetadata, config.isIncrementalTimelineSyncEnabled());
this.config = config;
this.schemaHelper = new RocksDBSchemaHelper(metaClient);
this.rocksDB = new RocksDBDAO(metaClient.getBasePath().toString(), config.getRocksdbBasePath());
init(metaClient, visibleActiveTimeline);
}
- public RocksDbBasedFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline,
+ public RocksDbBasedFileSystemView(HoodieTableMetadata tableMetadata, HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline,
List<StoragePathInfo> pathInfoList, FileSystemViewStorageConfig config) {
- this(metaClient, visibleActiveTimeline, config);
+ this(tableMetadata, metaClient, visibleActiveTimeline, config);
addFilesToView(pathInfoList);
}
@@ -94,7 +95,7 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
protected void init(HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline) {
schemaHelper.getAllColumnFamilies().forEach(rocksDB::addColumnFamily);
super.init(metaClient, visibleActiveTimeline);
- LOG.info("Created ROCKSDB based file-system view at " + config.getRocksdbBasePath());
+ LOG.info("Created ROCKSDB based file-system view at {}", config.getRocksdbBasePath());
}
@Override
@@ -109,7 +110,7 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
rocksDB.putInBatch(batch, schemaHelper.getColFamilyForPendingCompaction(),
schemaHelper.getKeyForPendingCompactionLookup(opPair.getValue().getFileGroupId()), opPair)
);
- LOG.info("Initializing pending compaction operations. Count=" + batch.count());
+ LOG.info("Initializing pending compaction operations. Count={}", batch.count());
});
}
@@ -152,7 +153,7 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
rocksDB.putInBatch(batch, schemaHelper.getColFamilyForPendingLogCompaction(),
schemaHelper.getKeyForPendingLogCompactionLookup(opPair.getValue().getFileGroupId()), opPair)
);
- LOG.info("Initializing pending Log compaction operations. Count=" + batch.count());
+ LOG.info("Initializing pending Log compaction operations. Count={}", batch.count());
});
}
@@ -275,8 +276,8 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
@Override
protected void storePartitionView(String partitionPath, List<HoodieFileGroup> fileGroups) {
- LOG.info("Resetting and adding new partition (" + partitionPath + ") to ROCKSDB based file-system view at "
- + config.getRocksdbBasePath() + ", Total file-groups=" + fileGroups.size());
+ LOG.info("Resetting and adding new partition ({}) to ROCKSDB based file-system view at {}, Total file-groups={}",
+ partitionPath, config.getRocksdbBasePath(), fileGroups.size());
String lookupKey = schemaHelper.getKeyForPartitionLookup(partitionPath);
rocksDB.delete(schemaHelper.getColFamilyForStoredPartitions(), lookupKey);
@@ -301,8 +302,8 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
// record that partition is loaded.
rocksDB.put(schemaHelper.getColFamilyForStoredPartitions(), lookupKey, Boolean.TRUE);
- LOG.info("Finished adding new partition (" + partitionPath + ") to ROCKSDB based file-system view at "
- + config.getRocksdbBasePath() + ", Total file-groups=" + fileGroups.size());
+ LOG.info("Finished adding new partition ({}}) to ROCKSDB based file-system view at {}, Total file-groups={}",
+ partitionPath, config.getRocksdbBasePath(), fileGroups.size());
}
@Override
@@ -320,7 +321,7 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
} else {
FileSlice oldSlice = oldSliceOption.get();
// First remove the file-slice
- LOG.info("Removing old Slice in DB. FS=" + oldSlice);
+ LOG.info("Removing old Slice in DB. FS={}", oldSlice);
rocksDB.deleteInBatch(batch, schemaHelper.getColFamilyForView(), schemaHelper.getKeyForSliceView(fg, oldSlice));
rocksDB.deleteInBatch(batch, schemaHelper.getColFamilyForView(), schemaHelper.getKeyForDataFileView(fg, oldSlice));
@@ -340,11 +341,11 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
deltaLogFiles.entrySet().stream().filter(e -> !logFiles.containsKey(e.getKey()))
.forEach(p -> newLogFiles.put(p.getKey(), p.getValue()));
newLogFiles.values().forEach(newFileSlice::addLogFile);
- LOG.info("Adding back new File Slice after add FS=" + newFileSlice);
+ LOG.info("Adding back new File Slice after add FS={}", newFileSlice);
return newFileSlice;
}
case REMOVE: {
- LOG.info("Removing old File Slice =" + fs);
+ LOG.info("Removing old File Slice ={}", fs);
FileSlice newFileSlice = new FileSlice(oldSlice.getFileGroupId(), oldSlice.getBaseInstantTime());
fs.getBaseFile().orElseGet(() -> {
oldSlice.getBaseFile().ifPresent(newFileSlice::setBaseFile);
@@ -355,7 +356,7 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
// Add remaining log files back
logFiles.values().forEach(newFileSlice::addLogFile);
if (newFileSlice.getBaseFile().isPresent() || (newFileSlice.getLogFiles().count() > 0)) {
- LOG.info("Adding back new file-slice after remove FS=" + newFileSlice);
+ LOG.info("Adding back new file-slice after remove FS={}", newFileSlice);
return newFileSlice;
}
return null;
@@ -404,7 +405,7 @@ public class RocksDbBasedFileSystemView extends IncrementalTimelineSyncFileSyste
rocksDB.putInBatch(batch, schemaHelper.getColFamilyForBootstrapBaseFile(),
schemaHelper.getKeyForBootstrapBaseFile(externalBaseFile.getFileGroupId()), externalBaseFile);
});
- LOG.info("Initializing external data file mapping. Count=" + batch.count());
+ LOG.info("Initializing external data file mapping. Count={}", batch.count());
});
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java
index 1f4eb3c1525..13456d0e435 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java
@@ -23,6 +23,8 @@ import org.apache.hudi.common.model.BootstrapBaseFileMapping;
import org.apache.hudi.common.model.CompactionOperation;
import org.apache.hudi.common.model.HoodieFileGroup;
import org.apache.hudi.common.model.HoodieFileGroupId;
+import org.apache.hudi.common.serialization.DefaultSerializer;
+import org.apache.hudi.common.serialization.HoodieFileGroupSerializer;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
@@ -30,6 +32,7 @@ import org.apache.hudi.common.util.DefaultSizeEstimator;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ExternalSpillableMap;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.storage.StoragePathInfo;
import org.slf4j.Logger;
@@ -59,9 +62,9 @@ public class SpillableMapBasedFileSystemView extends HoodieTableFileSystemView {
private final ExternalSpillableMap.DiskMapType diskMapType;
private final boolean isBitCaskDiskMapCompressionEnabled;
- public SpillableMapBasedFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline,
- FileSystemViewStorageConfig config, HoodieCommonConfig commonConfig) {
- super(config.isIncrementalTimelineSyncEnabled());
+ public SpillableMapBasedFileSystemView(HoodieTableMetadata tableMetadata, HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline,
+ FileSystemViewStorageConfig config, HoodieCommonConfig commonConfig) {
+ super(tableMetadata, config.isIncrementalTimelineSyncEnabled());
this.maxMemoryForFileGroupMap = config.getMaxMemoryForFileGroupMap();
this.maxMemoryForPendingCompaction = config.getMaxMemoryForPendingCompaction();
this.maxMemoryForPendingLogCompaction = config.getMaxMemoryForPendingLogCompaction();
@@ -69,29 +72,30 @@ public class SpillableMapBasedFileSystemView extends HoodieTableFileSystemView {
this.maxMemoryForReplaceFileGroups = config.getMaxMemoryForReplacedFileGroups();
this.maxMemoryForClusteringFileGroups = config.getMaxMemoryForPendingClusteringFileGroups();
this.baseStoreDir = config.getSpillableDir();
+ new File(baseStoreDir).mkdirs();
diskMapType = commonConfig.getSpillableDiskMapType();
isBitCaskDiskMapCompressionEnabled = commonConfig.isBitCaskDiskMapCompressionEnabled();
init(metaClient, visibleActiveTimeline);
}
- public SpillableMapBasedFileSystemView(HoodieTableMetaClient metaClient,
+ public SpillableMapBasedFileSystemView(HoodieTableMetadata tableMetadata,
+ HoodieTableMetaClient metaClient,
HoodieTimeline visibleActiveTimeline,
List<StoragePathInfo> pathInfoList,
FileSystemViewStorageConfig config,
HoodieCommonConfig commonConfig) {
- this(metaClient, visibleActiveTimeline, config, commonConfig);
+ this(tableMetadata, metaClient, visibleActiveTimeline, config, commonConfig);
addFilesToView(pathInfoList);
}
@Override
protected Map<String, List<HoodieFileGroup>> createPartitionToFileGroups() {
try {
- LOG.info("Creating Partition To File groups map using external spillable Map. Max Mem=" + maxMemoryForFileGroupMap
- + ", BaseDir=" + baseStoreDir);
- new File(baseStoreDir).mkdirs();
+ LOG.info("Creating Partition To File groups map using external spillable Map. Max Mem={}, BaseDir={}", maxMemoryForFileGroupMap, baseStoreDir);
+ closeFileGroupsMapIfPresent();
return (Map<String, List<HoodieFileGroup>>) (new ExternalSpillableMap<>(maxMemoryForFileGroupMap, baseStoreDir,
- new DefaultSizeEstimator(), new DefaultSizeEstimator<>(),
- diskMapType, isBitCaskDiskMapCompressionEnabled));
+ new DefaultSizeEstimator(), new HoodieFileGroupSizeEstimator(),
+ diskMapType, new HoodieFileGroupSerializer(), isBitCaskDiskMapCompressionEnabled));
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -101,12 +105,11 @@ public class SpillableMapBasedFileSystemView extends HoodieTableFileSystemView {
protected Map<HoodieFileGroupId, Pair<String, CompactionOperation>> createFileIdToPendingCompactionMap(
Map<HoodieFileGroupId, Pair<String, CompactionOperation>> fgIdToPendingCompaction) {
try {
- LOG.info("Creating Pending Compaction map using external spillable Map. Max Mem=" + maxMemoryForPendingCompaction
- + ", BaseDir=" + baseStoreDir);
- new File(baseStoreDir).mkdirs();
+ LOG.info("Creating Pending Compaction map using external spillable Map. Max Mem={}, BaseDir={}", maxMemoryForPendingCompaction, baseStoreDir);
+ closePendingCompactionMapIfPresent();
Map<HoodieFileGroupId, Pair<String, CompactionOperation>> pendingMap = new ExternalSpillableMap<>(
- maxMemoryForPendingCompaction, baseStoreDir, new DefaultSizeEstimator(), new DefaultSizeEstimator<>(),
- diskMapType, isBitCaskDiskMapCompressionEnabled);
+ maxMemoryForPendingCompaction, baseStoreDir, new DefaultSizeEstimator<>(), new DefaultSizeEstimator<>(),
+ diskMapType, new DefaultSerializer<>(), isBitCaskDiskMapCompressionEnabled);
pendingMap.putAll(fgIdToPendingCompaction);
return pendingMap;
} catch (IOException e) {
@@ -118,12 +121,11 @@ public class SpillableMapBasedFileSystemView extends HoodieTableFileSystemView {
protected Map<HoodieFileGroupId, Pair<String, CompactionOperation>> createFileIdToPendingLogCompactionMap(
Map<HoodieFileGroupId, Pair<String, CompactionOperation>> fgIdToPendingLogCompaction) {
try {
- LOG.info("Creating Pending Log Compaction map using external spillable Map. Max Mem=" + maxMemoryForPendingLogCompaction
- + ", BaseDir=" + baseStoreDir);
- new File(baseStoreDir).mkdirs();
+ LOG.info("Creating Pending Log Compaction map using external spillable Map. Max Mem={}, BaseDir={}", maxMemoryForPendingLogCompaction, baseStoreDir);
+ closePendingLogCompactionMapIfPresent();
Map<HoodieFileGroupId, Pair<String, CompactionOperation>> pendingMap = new ExternalSpillableMap<>(
maxMemoryForPendingLogCompaction, baseStoreDir, new DefaultSizeEstimator(), new DefaultSizeEstimator<>(),
- diskMapType, isBitCaskDiskMapCompressionEnabled);
+ diskMapType, new DefaultSerializer<>(), isBitCaskDiskMapCompressionEnabled);
pendingMap.putAll(fgIdToPendingLogCompaction);
return pendingMap;
} catch (IOException e) {
@@ -135,12 +137,11 @@ public class SpillableMapBasedFileSystemView extends HoodieTableFileSystemView {
protected Map<HoodieFileGroupId, BootstrapBaseFileMapping> createFileIdToBootstrapBaseFileMap(
Map<HoodieFileGroupId, BootstrapBaseFileMapping> fileGroupIdBootstrapBaseFileMap) {
try {
- LOG.info("Creating bootstrap base File Map using external spillable Map. Max Mem=" + maxMemoryForBootstrapBaseFile
- + ", BaseDir=" + baseStoreDir);
- new File(baseStoreDir).mkdirs();
+ LOG.info("Creating bootstrap base File Map using external spillable Map. Max Mem={}, BaseDir={}", maxMemoryForBootstrapBaseFile, baseStoreDir);
+ closeBootstrapFileMapIfPresent();
Map<HoodieFileGroupId, BootstrapBaseFileMapping> pendingMap = new ExternalSpillableMap<>(
maxMemoryForBootstrapBaseFile, baseStoreDir, new DefaultSizeEstimator(), new DefaultSizeEstimator<>(),
- diskMapType, isBitCaskDiskMapCompressionEnabled);
+ diskMapType, new DefaultSerializer<>(), isBitCaskDiskMapCompressionEnabled);
pendingMap.putAll(fileGroupIdBootstrapBaseFileMap);
return pendingMap;
} catch (IOException e) {
@@ -151,12 +152,11 @@ public class SpillableMapBasedFileSystemView extends HoodieTableFileSystemView {
@Override
protected Map<HoodieFileGroupId, HoodieInstant> createFileIdToReplaceInstantMap(final Map<HoodieFileGroupId, HoodieInstant> replacedFileGroups) {
try {
- LOG.info("Creating file group id to replace instant map using external spillable Map. Max Mem=" + maxMemoryForReplaceFileGroups
- + ", BaseDir=" + baseStoreDir);
- new File(baseStoreDir).mkdirs();
+ LOG.info("Creating file group id to replace instant map using external spillable Map. Max Mem={}, BaseDir={}", maxMemoryForReplaceFileGroups, baseStoreDir);
+ closeReplaceInstantsMapIfPresent();
Map<HoodieFileGroupId, HoodieInstant> pendingMap = new ExternalSpillableMap<>(
maxMemoryForReplaceFileGroups, baseStoreDir, new DefaultSizeEstimator(), new DefaultSizeEstimator<>(),
- diskMapType, isBitCaskDiskMapCompressionEnabled);
+ diskMapType, new DefaultSerializer<>(), isBitCaskDiskMapCompressionEnabled);
pendingMap.putAll(replacedFileGroups);
return pendingMap;
} catch (IOException e) {
@@ -167,12 +167,11 @@ public class SpillableMapBasedFileSystemView extends HoodieTableFileSystemView {
@Override
protected Map<HoodieFileGroupId, HoodieInstant> createFileIdToPendingClusteringMap(final Map<HoodieFileGroupId, HoodieInstant> fileGroupsInClustering) {
try {
- LOG.info("Creating file group id to clustering instant map using external spillable Map. Max Mem=" + maxMemoryForClusteringFileGroups
- + ", BaseDir=" + baseStoreDir);
- new File(baseStoreDir).mkdirs();
+ LOG.info("Creating file group id to clustering instant map using external spillable Map. Max Mem={}, BaseDir={}", maxMemoryForClusteringFileGroups, baseStoreDir);
+ closePendingClusteringMapIfPresent();
Map<HoodieFileGroupId, HoodieInstant> pendingMap = new ExternalSpillableMap<>(
maxMemoryForClusteringFileGroups, baseStoreDir, new DefaultSizeEstimator(), new DefaultSizeEstimator<>(),
- diskMapType, isBitCaskDiskMapCompressionEnabled);
+ diskMapType, new DefaultSerializer<>(), isBitCaskDiskMapCompressionEnabled);
pendingMap.putAll(fileGroupsInClustering);
return pendingMap;
} catch (IOException e) {
@@ -222,24 +221,48 @@ public class SpillableMapBasedFileSystemView extends HoodieTableFileSystemView {
@Override
public void close() {
+ closeFileGroupsMapIfPresent();
+ closePendingClusteringMapIfPresent();
+ closePendingCompactionMapIfPresent();
+ closePendingLogCompactionMapIfPresent();
+ closeBootstrapFileMapIfPresent();
+ closeReplaceInstantsMapIfPresent();
super.close();
- if (partitionToFileGroupsMap != null) {
- ((ExternalSpillableMap) partitionToFileGroupsMap).close();
- }
- if (fgIdToPendingClustering != null) {
- ((ExternalSpillableMap) fgIdToPendingClustering).close();
+ }
+
+ private void closeReplaceInstantsMapIfPresent() {
+ if (fgIdToReplaceInstants != null) {
+ ((ExternalSpillableMap) fgIdToReplaceInstants).close();
}
- if (fgIdToPendingCompaction != null) {
- ((ExternalSpillableMap) fgIdToPendingCompaction).close();
+ }
+
+ private void closeBootstrapFileMapIfPresent() {
+ if (fgIdToBootstrapBaseFile != null) {
+ ((ExternalSpillableMap) fgIdToBootstrapBaseFile).close();
}
+ }
+
+ private void closePendingLogCompactionMapIfPresent() {
if (fgIdToPendingLogCompaction != null) {
((ExternalSpillableMap) fgIdToPendingLogCompaction).close();
}
- if (fgIdToBootstrapBaseFile != null) {
- ((ExternalSpillableMap) fgIdToBootstrapBaseFile).close();
+ }
+
+ private void closePendingCompactionMapIfPresent() {
+ if (fgIdToPendingCompaction != null) {
+ ((ExternalSpillableMap) fgIdToPendingCompaction).close();
}
- if (fgIdToReplaceInstants != null) {
- ((ExternalSpillableMap) fgIdToReplaceInstants).close();
+ }
+
+ private void closePendingClusteringMapIfPresent() {
+ if (fgIdToPendingClustering != null) {
+ ((ExternalSpillableMap) fgIdToPendingClustering).close();
+ }
+ }
+
+ private void closeFileGroupsMapIfPresent() {
+ if (partitionToFileGroupsMap != null) {
+ ((ExternalSpillableMap) partitionToFileGroupsMap).close();
}
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/BitCaskDiskMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/BitCaskDiskMap.java
index 01794e6cf3b..1b7458f9e81 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/BitCaskDiskMap.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/BitCaskDiskMap.java
@@ -19,6 +19,7 @@
package org.apache.hudi.common.util.collection;
import org.apache.hudi.common.fs.SizeAwareDataOutputStream;
+import org.apache.hudi.common.serialization.CustomSerializer;
import org.apache.hudi.common.util.BufferedRandomAccessFile;
import org.apache.hudi.common.util.SerializationUtils;
import org.apache.hudi.common.util.SpillableMapUtils;
@@ -66,7 +67,7 @@ import static org.apache.hudi.common.util.BinaryUtil.generateChecksum;
* <p>
* Inspired by https://github.com/basho/bitcask
*/
-public final class BitCaskDiskMap<T extends Serializable, R extends Serializable> extends DiskMap<T, R> {
+public final class BitCaskDiskMap<T extends Serializable, R> extends DiskMap<T, R> {
public static final int BUFFER_SIZE = 128 * 1024; // 128 KB
private static final Logger LOG = LoggerFactory.getLogger(BitCaskDiskMap.class);
@@ -94,8 +95,9 @@ public final class BitCaskDiskMap<T extends Serializable, R extends Serializable
private final Queue<BufferedRandomAccessFile> openedAccessFiles = new ConcurrentLinkedQueue<>();
private final List<ClosableIterator<R>> iterators = new ArrayList<>();
+ private final CustomSerializer<R> valueSerializer;
- public BitCaskDiskMap(String baseFilePath, boolean isCompressionEnabled) throws IOException {
+ public BitCaskDiskMap(String baseFilePath, CustomSerializer<R> valueSerializer, boolean isCompressionEnabled) throws IOException {
super(baseFilePath, ExternalSpillableMap.DiskMapType.BITCASK.name());
this.valueMetadataMap = new ConcurrentHashMap<>();
this.isCompressionEnabled = isCompressionEnabled;
@@ -105,10 +107,7 @@ public final class BitCaskDiskMap<T extends Serializable, R extends Serializable
this.fileOutputStream = new FileOutputStream(writeOnlyFile, true);
this.writeOnlyFileHandle = new SizeAwareDataOutputStream(fileOutputStream, BUFFER_SIZE);
this.filePosition = new AtomicLong(0L);
- }
-
- public BitCaskDiskMap(String baseFilePath) throws IOException {
- this(baseFilePath, false);
+ this.valueSerializer = valueSerializer;
}
/**
@@ -158,7 +157,7 @@ public final class BitCaskDiskMap<T extends Serializable, R extends Serializable
*/
@Override
public Iterator<R> iterator() {
- ClosableIterator<R> iterator = new LazyFileIterable(filePath, valueMetadataMap, isCompressionEnabled).iterator();
+ ClosableIterator<R> iterator = new LazyFileIterable(filePath, valueMetadataMap, valueSerializer, isCompressionEnabled).iterator();
this.iterators.add(iterator);
return iterator;
}
@@ -169,7 +168,7 @@ public final class BitCaskDiskMap<T extends Serializable, R extends Serializable
@Override
public Iterator<R> iterator(Predicate<T> filter) {
Map<T, ValueMetadata> filteredValueMetadata = valueMetadataMap.entrySet().stream().filter(e -> filter.test(e.getKey())).collect(Collectors.toMap(Entry::getKey, Entry::getValue));
- ClosableIterator<R> iterator = new LazyFileIterable(filePath, filteredValueMetadata, isCompressionEnabled).iterator();
+ ClosableIterator<R> iterator = new LazyFileIterable(filePath, filteredValueMetadata, valueSerializer, isCompressionEnabled).iterator();
this.iterators.add(iterator);
return iterator;
}
@@ -212,16 +211,16 @@ public final class BitCaskDiskMap<T extends Serializable, R extends Serializable
}
private R get(ValueMetadata entry) {
- return get(entry, getRandomAccessFile(), isCompressionEnabled);
+ return get(entry, getRandomAccessFile(), valueSerializer, isCompressionEnabled);
}
- public static <R> R get(ValueMetadata entry, RandomAccessFile file, boolean isCompressionEnabled) {
+ public static <V> V get(ValueMetadata entry, RandomAccessFile file, CustomSerializer<V> serializer, boolean isCompressionEnabled) {
try {
byte[] bytesFromDisk = SpillableMapUtils.readBytesFromDisk(file, entry.getOffsetOfValue(), entry.getSizeOfValue());
if (isCompressionEnabled) {
- return SerializationUtils.deserialize(DISK_COMPRESSION_REF.get().decompressBytes(bytesFromDisk));
+ return serializer.deserialize(DISK_COMPRESSION_REF.get().decompressBytes(bytesFromDisk));
}
- return SerializationUtils.deserialize(bytesFromDisk);
+ return serializer.deserialize(bytesFromDisk);
} catch (IOException e) {
throw new HoodieIOException("Unable to readFromDisk Hoodie Record from disk", e);
}
@@ -229,8 +228,8 @@ public final class BitCaskDiskMap<T extends Serializable, R extends Serializable
private synchronized R put(T key, R value, boolean flush) {
try {
- byte[] val = isCompressionEnabled ? DISK_COMPRESSION_REF.get().compressBytes(SerializationUtils.serialize(value)) :
- SerializationUtils.serialize(value);
+ byte[] val = isCompressionEnabled ? DISK_COMPRESSION_REF.get().compressBytes(valueSerializer.serialize(value)) :
+ valueSerializer.serialize(value);
int valueSize = val.length;
long timestamp = System.currentTimeMillis();
this.valueMetadataMap.put(key,
@@ -319,7 +318,7 @@ public final class BitCaskDiskMap<T extends Serializable, R extends Serializable
@Override
public Stream<R> valueStream() {
final BufferedRandomAccessFile file = getRandomAccessFile();
- return valueMetadataMap.values().stream().sorted().sequential().map(valueMetaData -> (R) get(valueMetaData, file, isCompressionEnabled));
+ return valueMetadataMap.values().stream().sorted().sequential().map(valueMetaData -> (R) get(valueMetaData, file, valueSerializer, isCompressionEnabled));
}
@Override
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/DiskMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/DiskMap.java
index 917d749733f..e15ecc581f3 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/DiskMap.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/DiskMap.java
@@ -37,7 +37,7 @@ import java.util.stream.Stream;
* @param <T> The generic type of the keys
* @param <R> The generic type of the values
*/
-public abstract class DiskMap<T extends Serializable, R extends Serializable> implements Map<T, R>, KeyFilteringIterable<T, R> {
+public abstract class DiskMap<T extends Serializable, R> implements Map<T, R>, KeyFilteringIterable<T, R>, AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(DiskMap.class);
private static final String SUBFOLDER_PREFIX = "hudi";
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java
index 202cfc08554..72452c4b9d4 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java
@@ -18,6 +18,7 @@
package org.apache.hudi.common.util.collection;
+import org.apache.hudi.common.serialization.CustomSerializer;
import org.apache.hudi.common.util.SizeEstimator;
import org.apache.hudi.exception.HoodieIOException;
@@ -56,7 +57,7 @@ import java.util.stream.Stream;
* frequently and incur unnecessary disk writes.
*/
@NotThreadSafe
-public class ExternalSpillableMap<T extends Serializable, R extends Serializable> implements Map<T, R>, Serializable, Closeable, KeyFilteringIterable<T, R> {
+public class ExternalSpillableMap<T extends Serializable, R> implements Map<T, R>, Serializable, Closeable, KeyFilteringIterable<T, R> {
// Find the actual estimated payload size after inserting N records
private static final int NUMBER_OF_RECORDS_TO_ESTIMATE_PAYLOAD_SIZE = 100;
@@ -84,19 +85,11 @@ public class ExternalSpillableMap<T extends Serializable, R extends Serializable
private volatile long estimatedPayloadSize = 0;
// Base File Path
private final String baseFilePath;
+ // Serializer for the values
+ private final CustomSerializer<R> valueSerializer;
public ExternalSpillableMap(long maxInMemorySizeInBytes, String baseFilePath, SizeEstimator<T> keySizeEstimator,
- SizeEstimator<R> valueSizeEstimator) throws IOException {
- this(maxInMemorySizeInBytes, baseFilePath, keySizeEstimator, valueSizeEstimator, DiskMapType.BITCASK);
- }
-
- public ExternalSpillableMap(long maxInMemorySizeInBytes, String baseFilePath, SizeEstimator<T> keySizeEstimator,
- SizeEstimator<R> valueSizeEstimator, DiskMapType diskMapType) throws IOException {
- this(maxInMemorySizeInBytes, baseFilePath, keySizeEstimator, valueSizeEstimator, diskMapType, false);
- }
-
- public ExternalSpillableMap(long maxInMemorySizeInBytes, String baseFilePath, SizeEstimator<T> keySizeEstimator,
- SizeEstimator<R> valueSizeEstimator, DiskMapType diskMapType, boolean isCompressionEnabled) throws IOException {
+ SizeEstimator<R> valueSizeEstimator, DiskMapType diskMapType, CustomSerializer<R> valueSerializer, boolean isCompressionEnabled) throws IOException {
this.inMemoryMap = new HashMap<>();
this.baseFilePath = baseFilePath;
this.maxInMemorySizeInBytes = (long) Math.floor(maxInMemorySizeInBytes * SIZING_FACTOR_FOR_IN_MEMORY_MAP);
@@ -105,6 +98,7 @@ public class ExternalSpillableMap<T extends Serializable, R extends Serializable
this.valueSizeEstimator = valueSizeEstimator;
this.diskMapType = diskMapType;
this.isCompressionEnabled = isCompressionEnabled;
+ this.valueSerializer = valueSerializer;
}
private void initDiskBasedMap() {
@@ -114,11 +108,11 @@ public class ExternalSpillableMap<T extends Serializable, R extends Serializable
try {
switch (diskMapType) {
case ROCKS_DB:
- diskBasedMap = new RocksDbDiskMap<>(baseFilePath);
+ diskBasedMap = new RocksDbDiskMap<>(baseFilePath, valueSerializer);
break;
case BITCASK:
default:
- diskBasedMap = new BitCaskDiskMap<>(baseFilePath, isCompressionEnabled);
+ diskBasedMap = new BitCaskDiskMap<>(baseFilePath, valueSerializer, isCompressionEnabled);
}
} catch (IOException e) {
throw new HoodieIOException(e.getMessage(), e);
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/LazyFileIterable.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/LazyFileIterable.java
index 799aa3d4d56..763b1018ff2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/LazyFileIterable.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/LazyFileIterable.java
@@ -18,6 +18,7 @@
package org.apache.hudi.common.util.collection;
+import org.apache.hudi.common.serialization.CustomSerializer;
import org.apache.hudi.common.util.BufferedRandomAccessFile;
import org.apache.hudi.exception.HoodieException;
@@ -43,23 +44,21 @@ public class LazyFileIterable<T, R> implements Iterable<R> {
private final Map<T, BitCaskDiskMap.ValueMetadata> inMemoryMetadataOfSpilledData;
// Was compressions enabled for the values when inserted into the file/ map
private final boolean isCompressionEnabled;
+ private final CustomSerializer<R> serializer;
private transient Thread shutdownThread = null;
- public LazyFileIterable(String filePath, Map<T, BitCaskDiskMap.ValueMetadata> map) {
- this(filePath, map, false);
- }
-
- public LazyFileIterable(String filePath, Map<T, BitCaskDiskMap.ValueMetadata> map, boolean isCompressionEnabled) {
+ public LazyFileIterable(String filePath, Map<T, BitCaskDiskMap.ValueMetadata> map, CustomSerializer<R> serializer, boolean isCompressionEnabled) {
this.filePath = filePath;
this.inMemoryMetadataOfSpilledData = map;
this.isCompressionEnabled = isCompressionEnabled;
+ this.serializer = serializer;
}
@Override
public ClosableIterator<R> iterator() {
try {
- return new LazyFileIterator<>(filePath, inMemoryMetadataOfSpilledData);
+ return new LazyFileIterator<>(filePath, inMemoryMetadataOfSpilledData, serializer);
} catch (IOException io) {
throw new HoodieException("Unable to initialize iterator for file on disk", io);
}
@@ -73,10 +72,12 @@ public class LazyFileIterable<T, R> implements Iterable<R> {
private final String filePath;
private BufferedRandomAccessFile readOnlyFileHandle;
private final Iterator<Map.Entry<T, BitCaskDiskMap.ValueMetadata>> metadataIterator;
+ private final CustomSerializer<R> serializer;
- public LazyFileIterator(String filePath, Map<T, BitCaskDiskMap.ValueMetadata> map) throws IOException {
+ public LazyFileIterator(String filePath, Map<T, BitCaskDiskMap.ValueMetadata> map, CustomSerializer<R> serializer) throws IOException {
this.filePath = filePath;
this.readOnlyFileHandle = new BufferedRandomAccessFile(filePath, "r", BitCaskDiskMap.BUFFER_SIZE);
+ this.serializer = serializer;
readOnlyFileHandle.seek(0);
// sort the map in increasing order of offset of value so disk seek is only in one(forward) direction
@@ -102,7 +103,7 @@ public class LazyFileIterable<T, R> implements Iterable<R> {
throw new IllegalStateException("next() called on EOF'ed stream. File :" + filePath);
}
Map.Entry<T, BitCaskDiskMap.ValueMetadata> entry = this.metadataIterator.next();
- return BitCaskDiskMap.get(entry.getValue(), readOnlyFileHandle, isCompressionEnabled);
+ return BitCaskDiskMap.get(entry.getValue(), readOnlyFileHandle, serializer, isCompressionEnabled);
}
@Override
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBDAO.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBDAO.java
index c76e97734e9..bb78010892e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBDAO.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBDAO.java
@@ -18,6 +18,8 @@
package org.apache.hudi.common.util.collection;
+import org.apache.hudi.common.serialization.CustomSerializer;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.SerializationUtils;
@@ -47,9 +49,11 @@ import java.io.IOException;
import java.io.Serializable;
import java.net.URI;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@@ -69,11 +73,17 @@ public class RocksDBDAO {
private transient RocksDB rocksDB;
private boolean closed = false;
private final String rocksDBBasePath;
+ private final transient Map<String, CustomSerializer<?>> columnFamilySerializers;
private long totalBytesWritten;
public RocksDBDAO(String basePath, String rocksDBBasePath) {
+ this(basePath, rocksDBBasePath, new HashMap<>());
+ }
+
+ public RocksDBDAO(String basePath, String rocksDBBasePath, Map<String, CustomSerializer<?>> columnFamilySerializers) {
this.rocksDBBasePath =
String.format("%s/%s/%s", rocksDBBasePath, URI.create(basePath).getPath().replace(":","").replace("/", "_"), UUID.randomUUID().toString());
+ this.columnFamilySerializers = columnFamilySerializers;
init();
totalBytesWritten = 0L;
}
@@ -193,7 +203,7 @@ public class RocksDBDAO {
*/
public <T extends Serializable> void putInBatch(WriteBatch batch, String columnFamilyName, String key, T value) {
try {
- byte[] payload = serializePayload(value);
+ byte[] payload = serializePayload(columnFamilyName, value);
batch.put(managedHandlesMap.get(columnFamilyName), getUTF8Bytes(key), payload);
} catch (Exception e) {
throw new HoodieException(e);
@@ -209,11 +219,11 @@ public class RocksDBDAO {
* @param value Payload
* @param <T> Type of payload
*/
- public <K extends Serializable, T extends Serializable> void putInBatch(WriteBatch batch, String columnFamilyName,
+ public <K extends Serializable, T> void putInBatch(WriteBatch batch, String columnFamilyName,
K key, T value) {
try {
byte[] keyBytes = SerializationUtils.serialize(key);
- byte[] payload = serializePayload(value);
+ byte[] payload = serializePayload(columnFamilyName, value);
batch.put(managedHandlesMap.get(columnFamilyName), keyBytes, payload);
} catch (Exception e) {
throw new HoodieException(e);
@@ -230,7 +240,7 @@ public class RocksDBDAO {
*/
public <T extends Serializable> void put(String columnFamilyName, String key, T value) {
try {
- byte[] payload = serializePayload(value);
+ byte[] payload = serializePayload(columnFamilyName, value);
getRocksDB().put(managedHandlesMap.get(columnFamilyName), getUTF8Bytes(key), payload);
} catch (Exception e) {
throw new HoodieException(e);
@@ -245,9 +255,9 @@ public class RocksDBDAO {
* @param value Payload
* @param <T> Type of Payload
*/
- public <K extends Serializable, T extends Serializable> void put(String columnFamilyName, K key, T value) {
+ public <K extends Serializable, T> void put(String columnFamilyName, K key, T value) {
try {
- byte[] payload = serializePayload(value);
+ byte[] payload = serializePayload(columnFamilyName, value);
getRocksDB().put(managedHandlesMap.get(columnFamilyName), SerializationUtils.serialize(key), payload);
} catch (Exception e) {
throw new HoodieException(e);
@@ -345,7 +355,7 @@ public class RocksDBDAO {
ValidationUtils.checkArgument(!closed);
try {
byte[] val = getRocksDB().get(managedHandlesMap.get(columnFamilyName), key);
- return val == null ? null : SerializationUtils.deserialize(val);
+ return deserializePayload(columnFamilyName, val);
} catch (Exception e) {
throw new HoodieException(e);
}
@@ -367,7 +377,7 @@ public class RocksDBDAO {
it.seek(getUTF8Bytes(prefix));
while (it.isValid() && new String(it.key()).startsWith(prefix)) {
long beginTs = System.nanoTime();
- T val = SerializationUtils.deserialize(it.value());
+ T val = deserializePayload(columnFamilyName, it.value());
timeTakenMicro += ((System.nanoTime() - beginTs) / 1000);
results.add(Pair.of(new String(it.key()), val));
it.next();
@@ -386,8 +396,8 @@ public class RocksDBDAO {
* @param <T> Type of key stored
* @param <R> Type of value stored
*/
- public <T extends Serializable, R extends Serializable> Iterator<Pair<T, R>> iterator(String columnFamilyName) {
- return new IteratorWrapper<>(getRocksDB().newIterator(managedHandlesMap.get(columnFamilyName)));
+ public <T extends Serializable, R> Iterator<Pair<T, R>> iterator(String columnFamilyName) {
+ return new IteratorWrapper<>(getRocksDB().newIterator(managedHandlesMap.get(columnFamilyName)), getSerializerForColumnFamily(columnFamilyName));
}
/**
@@ -491,12 +501,25 @@ public class RocksDBDAO {
return totalBytesWritten;
}
- private <T extends Serializable> byte[] serializePayload(T value) throws IOException {
- byte[] payload = SerializationUtils.serialize(value);
+ private <T> byte[] serializePayload(String columnFamily, T value) throws IOException {
+ CustomSerializer<T> serializer = getSerializerForColumnFamily(columnFamily);
+ byte[] payload = serializer.serialize(value);
totalBytesWritten += payload.length;
return payload;
}
+ private <T> T deserializePayload(String columnFamily, byte[] value) {
+ CustomSerializer<T> serializer = getSerializerForColumnFamily(columnFamily);
+ if (value == null) {
+ return null;
+ }
+ return serializer.deserialize(value);
+ }
+
+ private <T> CustomSerializer<T> getSerializerForColumnFamily(String columnFamily) {
+ return (CustomSerializer<T>) columnFamilySerializers.computeIfAbsent(columnFamily, cf -> new DefaultSerializer<>());
+ }
+
private byte[] getKeyBytes(String key) {
return getUTF8Bytes(key);
}
@@ -519,9 +542,11 @@ public class RocksDBDAO {
private static class IteratorWrapper<T, R> implements Iterator<Pair<T, R>> {
private final RocksIterator iterator;
+ private final CustomSerializer<R> deserializer;
- public IteratorWrapper(final RocksIterator iterator) {
+ public IteratorWrapper(final RocksIterator iterator, final CustomSerializer<R> deserializer) {
this.iterator = iterator;
+ this.deserializer = deserializer;
iterator.seekToFirst();
}
@@ -536,7 +561,7 @@ public class RocksDBDAO {
throw new IllegalStateException("next() called on rocksDB with no more valid entries");
}
T key = SerializationUtils.deserialize(iterator.key());
- R val = SerializationUtils.deserialize(iterator.value());
+ R val = deserializer.deserialize(iterator.value());
iterator.next();
return Pair.of(key, val);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDbDiskMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDbDiskMap.java
index b6c41a4154e..a75ede6e6dd 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDbDiskMap.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDbDiskMap.java
@@ -18,6 +18,7 @@
package org.apache.hudi.common.util.collection;
+import org.apache.hudi.common.serialization.CustomSerializer;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieNotSupportedException;
@@ -28,6 +29,7 @@ import java.io.IOException;
import java.io.Serializable;
import java.util.AbstractMap;
import java.util.Collection;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
@@ -41,7 +43,7 @@ import java.util.stream.StreamSupport;
* This class provides a disk spillable only map implementation.
* All of the data is stored using the RocksDB implementation.
*/
-public final class RocksDbDiskMap<T extends Serializable, R extends Serializable> extends DiskMap<T, R> {
+public final class RocksDbDiskMap<T extends Serializable, R> extends DiskMap<T, R> {
// ColumnFamily allows partitioning data within RockDB, which allows
// independent configuration and faster deletes across partitions
// https://github.com/facebook/rocksdb/wiki/Column-Families
@@ -52,11 +54,13 @@ public final class RocksDbDiskMap<T extends Serializable, R extends Serializable
private static final Logger LOG = LoggerFactory.getLogger(RocksDbDiskMap.class);
// Stores the key and corresponding value's latest metadata spilled to disk
private final Set<T> keySet;
+ private final CustomSerializer<R> valueSerializer;
private RocksDBDAO rocksDb;
- public RocksDbDiskMap(String rocksDbStoragePath) throws IOException {
+ public RocksDbDiskMap(String rocksDbStoragePath, CustomSerializer<R> valueSerializer) throws IOException {
super(rocksDbStoragePath, ExternalSpillableMap.DiskMapType.ROCKS_DB.name());
this.keySet = new HashSet<>();
+ this.valueSerializer = valueSerializer;
}
@Override
@@ -174,7 +178,9 @@ public final class RocksDbDiskMap<T extends Serializable, R extends Serializable
if (null == rocksDb) {
synchronized (this) {
if (null == rocksDb) {
- rocksDb = new RocksDBDAO(ROCKSDB_COL_FAMILY, diskMapPath);
+ Map<String, CustomSerializer<?>> serializerMap = new HashMap<>(4);
+ serializerMap.put(ROCKSDB_COL_FAMILY, valueSerializer);
+ rocksDb = new RocksDBDAO(ROCKSDB_COL_FAMILY, diskMapPath, serializerMap);
rocksDb.addColumnFamily(ROCKSDB_COL_FAMILY);
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
index efafbad5a08..bd0f4904d29 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/FileSystemBackedTableMetadata.java
@@ -48,6 +48,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
@@ -326,4 +327,24 @@ public class FileSystemBackedTableMetadata extends AbstractHoodieTableMetadata {
public int getNumFileGroupsForPartition(MetadataPartitionType partition) {
throw new HoodieMetadataException("Unsupported operation: getNumFileGroupsForPartition");
}
+
+ @Override
+ public Map<Pair<String, StoragePath>, List<StoragePathInfo>> listPartitions(List<Pair<String, StoragePath>> partitionPathList) throws IOException {
+ Map<Pair<String, StoragePath>, List<StoragePathInfo>> pathInfoMap = new HashMap<>();
+
+ for (Pair<String, StoragePath> partitionPair : partitionPathList) {
+ StoragePath absolutePartitionPath = partitionPair.getRight();
+ try {
+ pathInfoMap.put(partitionPair, getStorage().listDirectEntries(absolutePartitionPath));
+ } catch (IOException e) {
+ if (!getStorage().exists(absolutePartitionPath)) {
+ pathInfoMap.put(partitionPair, Collections.emptyList());
+ } else {
+ // in case the partition path was created by another caller
+ pathInfoMap.put(partitionPair, getStorage().listDirectEntries(absolutePartitionPath));
+ }
+ }
+ }
+ return pathInfoMap;
+ }
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
index 83c2fd20560..87433319719 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
@@ -54,6 +54,7 @@ import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.io.storage.HoodieSeekingFileReader;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.util.Transient;
import org.apache.avro.Schema;
@@ -72,6 +73,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
import java.util.stream.Collectors;
import static org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FULL_SCAN_LOG_FILES;
@@ -81,7 +83,7 @@ import static org.apache.hudi.common.util.ValidationUtils.checkState;
import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_BLOOM_FILTERS;
import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS;
import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_FILES;
-import static org.apache.hudi.metadata.HoodieTableMetadataUtil.getFileSystemView;
+import static org.apache.hudi.metadata.HoodieTableMetadataUtil.getFileSystemViewForMetadataTable;
/**
* Table metadata provided by an internal DFS backed Hudi metadata table.
@@ -135,16 +137,15 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
.setStorage(storage)
.setBasePath(metadataBasePath)
.build();
- this.metadataFileSystemView = getFileSystemView(metadataMetaClient);
this.metadataTableConfig = metadataMetaClient.getTableConfig();
} catch (TableNotFoundException e) {
- LOG.warn("Metadata table was not found at path " + metadataBasePath);
+ LOG.warn("Metadata table was not found at path {}", metadataBasePath);
this.isMetadataTableInitialized = false;
this.metadataMetaClient = null;
this.metadataFileSystemView = null;
this.metadataTableConfig = null;
} catch (Exception e) {
- LOG.error("Failed to initialize metadata table at path " + metadataBasePath, e);
+ LOG.error("Failed to initialize metadata table at path {}", metadataBasePath, e);
this.isMetadataTableInitialized = false;
this.metadataMetaClient = null;
this.metadataFileSystemView = null;
@@ -206,7 +207,7 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
// to scan all file-groups for all key-prefixes as each of these might contain some
// records matching the key-prefix
List<FileSlice> partitionFileSlices = partitionFileSliceMap.computeIfAbsent(partitionName,
- k -> HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(metadataMetaClient, metadataFileSystemView, partitionName));
+ k -> HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(metadataMetaClient, getMetadataFileSystemView(), partitionName));
checkState(!partitionFileSlices.isEmpty(), "Number of file slices for partition " + partitionName + " should be > 0");
return (shouldLoadInMemory ? HoodieListData.lazy(partitionFileSlices) :
@@ -236,8 +237,8 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
Map<String, HoodieRecord<HoodieMetadataPayload>> mergedRecords =
readFromBaseAndMergeWithLogRecords(baseFileReader, sortedKeyPrefixes, fullKeys, logRecords, timings, partitionName);
- LOG.debug(String.format("Metadata read for %s keys took [baseFileRead, logMerge] %s ms",
- sortedKeyPrefixes.size(), timings));
+ LOG.debug("Metadata read for {} keys took [baseFileRead, logMerge] {} ms",
+ sortedKeyPrefixes.size(), timings);
return mergedRecords.values().iterator();
} catch (IOException ioe) {
@@ -258,7 +259,7 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
// Load the file slices for the partition. Each file slice is a shard which saves a portion of the keys.
List<FileSlice> partitionFileSlices = partitionFileSliceMap.computeIfAbsent(partitionName,
- k -> HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(metadataMetaClient, metadataFileSystemView, partitionName));
+ k -> HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(metadataMetaClient, getMetadataFileSystemView(), partitionName));
final int numFileSlices = partitionFileSlices.size();
checkState(numFileSlices > 0, "Number of file slices for partition " + partitionName + " should be > 0");
@@ -305,7 +306,7 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
Map<String, List<HoodieRecord<HoodieMetadataPayload>>> result;
// Load the file slices for the partition. Each file slice is a shard which saves a portion of the keys.
List<FileSlice> partitionFileSlices = partitionFileSliceMap.computeIfAbsent(partitionName,
- k -> HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(metadataMetaClient, metadataFileSystemView, partitionName));
+ k -> HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(metadataMetaClient, getMetadataFileSystemView(), partitionName));
final int numFileSlices = partitionFileSlices.size();
checkState(numFileSlices > 0, "Number of file slices for partition " + partitionName + " should be > 0");
@@ -614,8 +615,8 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
.getReaderFactory(HoodieRecordType.AVRO)
.getFileReader(DEFAULT_HUDI_CONFIG_FOR_READER, baseFilePath);
baseFileOpenMs = timer.endTimer();
- LOG.info(String.format("Opened metadata base file from %s at instant %s in %d ms", baseFilePath,
- baseFile.get().getCommitTime(), baseFileOpenMs));
+ LOG.info("Opened metadata base file from {} at instant {} in {} ms", baseFilePath,
+ baseFile.get().getCommitTime(), baseFileOpenMs);
} else {
baseFileReader = null;
baseFileOpenMs = 0L;
@@ -669,8 +670,8 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
.build();
Long logScannerOpenMs = timer.endTimer();
- LOG.info(String.format("Opened %d metadata log files (dataset instant=%s, metadata instant=%s) in %d ms",
- sortedLogFilePaths.size(), getLatestDataInstantTime(), latestMetadataInstantTime, logScannerOpenMs));
+ LOG.info("Opened {} metadata log files (dataset instant={}, metadata instant={}) in {} ms",
+ sortedLogFilePaths.size(), getLatestDataInstantTime(), latestMetadataInstantTime, logScannerOpenMs);
return Pair.of(logRecordScanner, logScannerOpenMs);
}
@@ -744,6 +745,9 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
}
public HoodieTableFileSystemView getMetadataFileSystemView() {
+ if (metadataFileSystemView == null) {
+ metadataFileSystemView = getFileSystemViewForMetadataTable(metadataMetaClient);
+ }
return metadataFileSystemView;
}
@@ -780,8 +784,10 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
dataMetaClient.reloadActiveTimeline();
if (metadataMetaClient != null) {
metadataMetaClient.reloadActiveTimeline();
- metadataFileSystemView.close();
- metadataFileSystemView = getFileSystemView(metadataMetaClient);
+ if (metadataFileSystemView != null) {
+ metadataFileSystemView.close();
+ }
+ metadataFileSystemView = null;
}
// the cached reader has max instant time restriction, they should be cleared
// because the metadata timeline may have changed.
@@ -793,10 +799,25 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
public int getNumFileGroupsForPartition(MetadataPartitionType partition) {
partitionFileSliceMap.computeIfAbsent(partition.getPartitionPath(),
k -> HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(metadataMetaClient,
- metadataFileSystemView, partition.getPartitionPath()));
+ getMetadataFileSystemView(), partition.getPartitionPath()));
return partitionFileSliceMap.get(partition.getPartitionPath()).size();
}
+ @Override
+ public Map<Pair<String, StoragePath>, List<StoragePathInfo>> listPartitions(List<Pair<String, StoragePath>> partitionPathList) throws IOException {
+ Map<String, Pair<String, StoragePath>> absoluteToPairMap = partitionPathList.stream()
+ .collect(Collectors.toMap(
+ pair -> pair.getRight().toString(),
+ Function.identity()
+ ));
+ return getAllFilesInPartitions(
+ partitionPathList.stream().map(pair -> pair.getRight().toString()).collect(Collectors.toList()))
+ .entrySet().stream().collect(Collectors.toMap(
+ entry -> absoluteToPairMap.get(entry.getKey()),
+ Map.Entry::getValue
+ ));
+ }
+
@Override
public Map<String, Set<String>> getSecondaryIndexRecords(List<String> keys, String partitionName) {
if (keys.isEmpty()) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
deleted file mode 100644
index b75e4dd358b..00000000000
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.metadata;
-
-import org.apache.hudi.common.config.HoodieMetadataConfig;
-import org.apache.hudi.common.engine.HoodieEngineContext;
-import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.timeline.HoodieTimeline;
-import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
-import org.apache.hudi.common.util.collection.Pair;
-import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.StoragePathInfo;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-/**
- * {@code HoodieTableFileSystemView} implementation that retrieved partition listings from the Metadata Table.
- */
-public class HoodieMetadataFileSystemView extends HoodieTableFileSystemView {
-
- private final HoodieTableMetadata tableMetadata;
-
- public HoodieMetadataFileSystemView(HoodieTableMetaClient metaClient,
- HoodieTimeline visibleActiveTimeline,
- HoodieTableMetadata tableMetadata) {
- super(metaClient, visibleActiveTimeline);
- this.tableMetadata = tableMetadata;
- }
-
- public HoodieMetadataFileSystemView(HoodieEngineContext engineContext,
- HoodieTableMetaClient metaClient,
- HoodieTimeline visibleActiveTimeline,
- HoodieMetadataConfig metadataConfig) {
- this(metaClient, visibleActiveTimeline, HoodieTableMetadata.create(
- engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePath().toString(), true));
- }
-
- /**
- * Return all the files in the partition by reading from the Metadata Table.
- *
- * @param partitionPath The absolute path of the partition
- * @throws IOException
- */
- @Override
- protected List<StoragePathInfo> listPartition(StoragePath partitionPath) throws IOException {
- return tableMetadata.getAllFilesInPartition(partitionPath);
- }
-
- @Override
- protected List<String> getAllPartitionPaths() throws IOException {
- return tableMetadata.getAllPartitionPaths();
- }
-
- @Override
- protected Map<Pair<String, StoragePath>, List<StoragePathInfo>> listPartitions(
- List<Pair<String, StoragePath>> partitionPathList) throws IOException {
- Map<String, Pair<String, StoragePath>> absoluteToPairMap = partitionPathList.stream()
- .collect(Collectors.toMap(
- pair -> pair.getRight().toString(),
- Function.identity()
- ));
- return tableMetadata.getAllFilesInPartitions(
- partitionPathList.stream().map(pair -> pair.getRight().toString()).collect(Collectors.toList()))
- .entrySet().stream().collect(Collectors.toMap(
- entry -> absoluteToPairMap.get(entry.getKey()),
- Map.Entry::getValue
- ));
- }
-
- @Override
- public void reset() {
- try {
- writeLock.lock();
- clear();
- // Initialize with new Hoodie timeline.
- init(metaClient, getTimeline());
- tableMetadata.reset();
- } finally {
- writeLock.unlock();
- }
- }
-
- @Override
- public void sync() {
- try {
- writeLock.lock();
- maySyncIncrementally();
- tableMetadata.reset();
- } finally {
- writeLock.unlock();
- }
- }
-
- @Override
- public void close() {
- try {
- tableMetadata.close();
- } catch (Exception e) {
- throw new HoodieException("Error closing metadata file system view.", e);
- }
- }
-}
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
index e7392a8f1ac..49931975691 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
@@ -18,6 +18,7 @@
package org.apache.hudi.metadata;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -91,8 +92,9 @@ public class HoodieMetadataMetrics implements Serializable {
public Map<String, String> getStats(boolean detailed, HoodieTableMetaClient metaClient, HoodieTableMetadata metadata, Set<String> metadataPartitions) {
try {
- HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline());
- return getStats(fsView, detailed, metadata, metadataPartitions);
+ HoodieTableFileSystemView fileSystemView =
+ HoodieTableFileSystemView.fileListingBasedFileSystemView(new HoodieLocalEngineContext(metaClient.getStorageConf()), metaClient, metaClient.getActiveTimeline());
+ return getStats(fileSystemView, detailed, metadata, metadataPartitions);
} catch (IOException ioe) {
throw new HoodieIOException("Unable to get metadata stats.", ioe);
}
@@ -157,7 +159,7 @@ public class HoodieMetadataMetrics implements Serializable {
}
protected void incrementMetric(String action, long value) {
- LOG.debug(String.format("Updating metadata metrics (%s=%d) in %s", action, value, metricsRegistry));
+ LOG.debug("Updating metadata metrics ({}={}) in {}", action, value, metricsRegistry);
Option<HoodieGauge<Long>> gaugeOpt = metrics.registerGauge(action);
gaugeOpt.ifPresent(gauge -> gauge.setValue(gauge.getValue() + value));
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
index af11bfaf6e8..47db4b33736 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
@@ -305,4 +305,11 @@ public interface HoodieTableMetadata extends Serializable, AutoCloseable {
* Returns the number of shards in a metadata table partition.
*/
int getNumFileGroupsForPartition(MetadataPartitionType partition);
+
+ /**
+ * @param partitionPathList A list of pairs of the relative and absolute paths of the partitions.
+ * @return all the files from the partitions.
+ * @throws IOException upon error.
+ */
+ Map<Pair<String, StoragePath>, List<StoragePathInfo>> listPartitions(List<Pair<String, StoragePath>> partitionPathList) throws IOException;
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
index fe036770e56..1b35438dcb3 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
@@ -45,6 +45,7 @@ import org.apache.hudi.common.data.HoodieData;
import org.apache.hudi.common.data.HoodiePairData;
import org.apache.hudi.common.engine.EngineType;
import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.function.SerializableBiFunction;
import org.apache.hudi.common.function.SerializablePairFunction;
@@ -1385,7 +1386,7 @@ public class HoodieTableMetadataUtil {
* @param metaClient - Metadata table meta client
* @return Filesystem view for the metadata table
*/
- public static HoodieTableFileSystemView getFileSystemView(HoodieTableMetaClient metaClient) {
+ public static HoodieTableFileSystemView getFileSystemViewForMetadataTable(HoodieTableMetaClient metaClient) {
// If there are no commits on the metadata table then the table's
// default FileSystemView will not return any file slices even
// though we may have initialized them.
@@ -1396,7 +1397,8 @@ public class HoodieTableMetadataUtil {
metaClient.createNewInstantTime(false));
timeline = factory.createDefaultTimeline(Stream.of(instant), metaClient.getActiveTimeline()::getInstantDetails);
}
- return new HoodieTableFileSystemView(metaClient, timeline);
+ HoodieEngineContext engineContext = new HoodieLocalEngineContext(metaClient.getStorageConf());
+ return HoodieTableFileSystemView.fileListingBasedFileSystemView(engineContext, metaClient, timeline);
}
/**
@@ -1416,7 +1418,7 @@ public class HoodieTableMetadataUtil {
boolean mergeFileSlices) {
HoodieTableFileSystemView fsView = null;
try {
- fsView = fileSystemView.orElseGet(() -> getFileSystemView(metaClient));
+ fsView = fileSystemView.orElseGet(() -> getFileSystemViewForMetadataTable(metaClient));
Stream<FileSlice> fileSliceStream;
if (mergeFileSlices) {
if (metaClient.getActiveTimeline().filterCompletedInstants().lastInstant().isPresent()) {
@@ -1451,7 +1453,7 @@ public class HoodieTableMetadataUtil {
String partition) {
HoodieTableFileSystemView fsView = null;
try {
- fsView = fileSystemView.orElseGet(() -> getFileSystemView(metaClient));
+ fsView = fileSystemView.orElseGet(() -> getFileSystemViewForMetadataTable(metaClient));
Stream<FileSlice> fileSliceStream = fsView.getLatestFileSlicesIncludingInflight(partition);
return fileSliceStream
.sorted(Comparator.comparing(FileSlice::getFileId))
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java b/hudi-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java
index c4530c96359..ef8bf4ec06a 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java
@@ -36,6 +36,7 @@ import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.hudi.common.table.log.HoodieFileSliceReader;
import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner;
import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.HoodieRecordUtils;
@@ -99,7 +100,7 @@ public class SecondaryIndexRecordGenerationUtils {
String instantTime,
HoodieIndexDefinition indexDefinition,
HoodieMetadataConfig metadataConfig,
- HoodieMetadataFileSystemView fsView,
+ HoodieTableFileSystemView fsView,
HoodieTableMetaClient dataMetaClient,
HoodieEngineContext engineContext,
EngineType engineType) {
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestDefaultSerializer.java b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestDefaultSerializer.java
new file mode 100644
index 00000000000..42d53bd3698
--- /dev/null
+++ b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestDefaultSerializer.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.serialization;
+
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieBaseFile;
+import org.apache.hudi.common.model.HoodieFileGroupId;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.versioning.v1.InstantComparatorV1;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class TestDefaultSerializer {
+
+ @Test
+ void roundTripValidation() throws IOException {
+ String partition = "1";
+ String fileId1 = UUID.randomUUID().toString();
+ HoodieInstant instant1 = new HoodieInstant(HoodieInstant.State.COMPLETED, "commit", "001", InstantComparatorV1.REQUESTED_TIME_BASED_COMPARATOR);
+ FileSlice fileSlice = new FileSlice(new HoodieFileGroupId(partition, fileId1), instant1.requestedTime(),
+ new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant1.requestedTime(), "1-0-1", fileId1, "parquet")), Collections.emptyList());
+
+ DefaultSerializer<FileSlice> serializer = new DefaultSerializer<>();
+ byte[] serializedValue = serializer.serialize(fileSlice);
+ FileSlice output = serializer.deserialize(serializedValue);
+ // validate round trip serialization working
+ assertEquals(fileSlice, output);
+ }
+}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileGroupSerializer.java b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileGroupSerializer.java
new file mode 100644
index 00000000000..245af9fc688
--- /dev/null
+++ b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileGroupSerializer.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.serialization;
+
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieBaseFile;
+import org.apache.hudi.common.model.HoodieFileGroup;
+import org.apache.hudi.common.model.HoodieFileGroupId;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.versioning.v1.InstantComparatorV1;
+import org.apache.hudi.common.testutils.MockHoodieTimeline;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+
+class TestHoodieFileGroupSerializer {
+
+ @Test
+ void serializerReusesTimelineInstance() throws IOException {
+ // file groups 1 and 2 reference the same instance
+ String partition = "1";
+ String fileId1 = UUID.randomUUID().toString();
+ String fileId2 = UUID.randomUUID().toString();
+ HoodieInstant instant1 = new HoodieInstant(HoodieInstant.State.COMPLETED, "commit", "001", InstantComparatorV1.REQUESTED_TIME_BASED_COMPARATOR);
+ FileSlice fileSlice1 = new FileSlice(new HoodieFileGroupId(partition, fileId1), instant1.requestedTime(),
+ new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant1.requestedTime(), "1-0-1", fileId1, "parquet")), Collections.emptyList());
+ FileSlice fileSlice2 = new FileSlice(new HoodieFileGroupId(partition, fileId2), instant1.requestedTime(),
+ new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant1.requestedTime(), "1-0-1", fileId2, "parquet")), Collections.emptyList());
+ HoodieTimeline mockTimeline1 = new MockHoodieTimeline(Collections.singletonList(instant1));
+
+ HoodieFileGroup hoodieFileGroup1 = new HoodieFileGroup(partition, fileId1, mockTimeline1);
+ hoodieFileGroup1.addFileSlice(fileSlice1);
+ HoodieFileGroup hoodieFileGroup2 = new HoodieFileGroup(partition, fileId2, mockTimeline1);
+ hoodieFileGroup2.addFileSlice(fileSlice2);
+
+ // file group 3 references a new instance
+ String fileId3 = UUID.randomUUID().toString();
+ HoodieInstant instant2 = new HoodieInstant(HoodieInstant.State.COMPLETED, "commit", "002", InstantComparatorV1.REQUESTED_TIME_BASED_COMPARATOR);
+ HoodieTimeline mockTimeline2 = new MockHoodieTimeline(Collections.singletonList(instant2));
+ FileSlice fileSlice3 = new FileSlice(new HoodieFileGroupId(partition, fileId3), instant1.requestedTime(),
+ new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant2.requestedTime(), "1-0-1", fileId3, "parquet")), Collections.emptyList());
+ HoodieFileGroup hoodieFileGroup3 = new HoodieFileGroup(partition, fileId3, mockTimeline2);
+ hoodieFileGroup3.addFileSlice(fileSlice3);
+
+ List<HoodieFileGroup> inputs = Arrays.asList(hoodieFileGroup1, hoodieFileGroup2, hoodieFileGroup3);
+ HoodieFileGroupSerializer serializer = new HoodieFileGroupSerializer();
+ byte[] serializedValue = serializer.serialize(inputs);
+ List<HoodieFileGroup> outputs = serializer.deserialize(serializedValue);
+ // validate round trip serialization working
+ assertEquals(inputs, outputs);
+ // validate that the timeline instances are the same object, not just equivalent
+ assertSame(mockTimeline1, outputs.get(0).getTimeline());
+ assertSame(mockTimeline1, outputs.get(1).getTimeline());
+ assertSame(mockTimeline2, outputs.get(2).getTimeline());
+ }
+}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestLogFileIterator.java b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestLogFileIterator.java
index f0138ecc5f5..0249aac0e5b 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestLogFileIterator.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestLogFileIterator.java
@@ -22,6 +22,7 @@ package org.apache.hudi.common.table.log;
import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.util.DefaultSizeEstimator;
import org.apache.hudi.common.util.HoodieRecordSizeEstimator;
@@ -104,6 +105,7 @@ public class TestLogFileIterator {
new DefaultSizeEstimator(),
new HoodieRecordSizeEstimator(HoodieTestDataGenerator.AVRO_SCHEMA),
ExternalSpillableMap.DiskMapType.BITCASK,
+ new DefaultSerializer<>(),
false
);
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java
index fb1eddd83fb..2335b2c632f 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java
@@ -31,6 +31,7 @@ import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.engine.HoodieReaderContext;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.TableSchemaResolver;
@@ -186,7 +187,7 @@ public abstract class TestHoodieFileGroupReaderBase<T> {
for (Boolean isCompressionEnabled : new boolean[] {true, false}) {
try (ExternalSpillableMap<Serializable, Pair<Option<T>, Map<String, Object>>> spillableMap =
new ExternalSpillableMap<>(16L, baseMapPath, new DefaultSizeEstimator(),
- new HoodieRecordSizeEstimator(avroSchema), diskMapType, isCompressionEnabled)) {
+ new HoodieRecordSizeEstimator(avroSchema), diskMapType, new DefaultSerializer<>(), isCompressionEnabled)) {
Long position = 0L;
for (T record : records) {
String recordKey = readerContext.getRecordKey(record, avroSchema);
@@ -273,7 +274,9 @@ public abstract class TestHoodieFileGroupReaderBase<T> {
HoodieEngineContext engineContext = new HoodieLocalEngineContext(storageConf);
HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().build();
FileSystemViewManager viewManager = FileSystemViewManager.createViewManager(
- engineContext, FileSystemViewStorageConfig.newBuilder().build(),
+ engineContext,
+ metadataConfig,
+ FileSystemViewStorageConfig.newBuilder().build(),
HoodieCommonConfig.newBuilder().build(),
mc -> HoodieTableMetadata.create(
engineContext, mc.getStorage(), metadataConfig, tablePath));
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieFileGroupSizeEstimator.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieFileGroupSizeEstimator.java
new file mode 100644
index 00000000000..90ad842125d
--- /dev/null
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieFileGroupSizeEstimator.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.table.view;
+
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieBaseFile;
+import org.apache.hudi.common.model.HoodieFileGroup;
+import org.apache.hudi.common.model.HoodieFileGroupId;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Stream;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class TestHoodieFileGroupSizeEstimator {
+ @Test
+ void estimatorSkipsTimeline() {
+ HoodieFileGroup fileGroup1 = mock(HoodieFileGroup.class);
+ HoodieFileGroup fileGroup2 = mock(HoodieFileGroup.class);
+
+ // setup mocks
+ HoodieFileGroupId fileGroupId = new HoodieFileGroupId("path1", UUID.randomUUID().toString());
+ List<FileSlice> fileSlices = Collections.singletonList(new FileSlice(fileGroupId, "001",
+ new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName("001", "1-0-1", fileGroupId.getFileId(), "parquet")), Collections.emptyList()));
+ when(fileGroup1.getFileGroupId()).thenReturn(fileGroupId);
+ when(fileGroup1.getAllFileSlices()).thenReturn(fileSlices.stream());
+
+ when(fileGroup2.getFileGroupId()).thenReturn(new HoodieFileGroupId("path2", UUID.randomUUID().toString()));
+ when(fileGroup2.getAllFileSlices()).thenReturn(Stream.empty());
+
+ long result = new HoodieFileGroupSizeEstimator().sizeEstimate(Arrays.asList(fileGroup1, fileGroup2));
+ Assertions.assertTrue(result > 0);
+ verify(fileGroup1, never()).getTimeline();
+ verify(fileGroup2, never()).getTimeline();
+ }
+}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
index 1bf9c8a825f..2c1667a8c5b 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
@@ -18,7 +18,8 @@
package org.apache.hudi.common.table.view;
-import org.apache.hudi.common.function.SerializableSupplier;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.function.SerializableFunctionUnchecked;
import org.apache.hudi.common.model.CompactionOperation;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieBaseFile;
@@ -68,6 +69,8 @@ import static org.mockito.Mockito.when;
*/
@ExtendWith(MockitoExtension.class)
public class TestPriorityBasedFileSystemView {
+ @Mock
+ private HoodieEngineContext engineContext;
@Mock
private SyncableFileSystemView primary;
@@ -75,7 +78,7 @@ public class TestPriorityBasedFileSystemView {
@Mock
private SyncableFileSystemView secondary;
@Mock
- private SerializableSupplier<SyncableFileSystemView> secondaryViewSupplier;
+ private SerializableFunctionUnchecked<HoodieEngineContext, SyncableFileSystemView> secondaryViewCreator;
@InjectMocks
private PriorityBasedFileSystemView fsView;
@@ -85,7 +88,7 @@ public class TestPriorityBasedFileSystemView {
@BeforeEach
public void setUp() {
- fsView = new PriorityBasedFileSystemView(primary, secondaryViewSupplier);
+ fsView = new PriorityBasedFileSystemView(primary, secondaryViewCreator, engineContext);
testBaseFileStream = Stream.of(new HoodieBaseFile("test"));
testFileSliceStream = Stream.of(new FileSlice("2020-01-01", "20:20",
"file0001" + HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension()));
@@ -110,10 +113,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getLatestBaseFiles()).thenReturn(testBaseFileStream);
actual = fsView.getLatestBaseFiles();
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestBaseFiles()).thenThrow(new RuntimeException());
when(secondary.getLatestBaseFiles()).thenReturn(testBaseFileStream);
actual = fsView.getLatestBaseFiles();
@@ -142,7 +145,7 @@ public class TestPriorityBasedFileSystemView {
Stream<HoodieBaseFile> expected = testBaseFileStream;
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestBaseFiles()).thenThrow(new RuntimeException(new HttpResponseException(400, "Bad Request")));
when(secondary.getLatestBaseFiles()).thenReturn(testBaseFileStream);
actual = fsView.getLatestBaseFiles();
@@ -166,10 +169,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getLatestBaseFiles(partitionPath)).thenReturn(testBaseFileStream);
actual = fsView.getLatestBaseFiles(partitionPath);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestBaseFiles(partitionPath)).thenThrow(new RuntimeException());
when(secondary.getLatestBaseFiles(partitionPath)).thenReturn(testBaseFileStream);
actual = fsView.getLatestBaseFiles(partitionPath);
@@ -198,10 +201,10 @@ public class TestPriorityBasedFileSystemView {
.thenReturn(testBaseFileStream);
actual = fsView.getLatestBaseFilesBeforeOrOn(partitionPath, maxCommitTime);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestBaseFilesBeforeOrOn(partitionPath, maxCommitTime))
.thenThrow(new RuntimeException());
when(secondary.getLatestBaseFilesBeforeOrOn(partitionPath, maxCommitTime))
@@ -233,10 +236,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getLatestBaseFile(partitionPath, fileID)).thenReturn(expected);
actual = fsView.getLatestBaseFile(partitionPath, fileID);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestBaseFile(partitionPath, fileID)).thenThrow(new RuntimeException());
when(secondary.getLatestBaseFile(partitionPath, fileID)).thenReturn(expected);
actual = fsView.getLatestBaseFile(partitionPath, fileID);
@@ -265,10 +268,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getBaseFileOn(partitionPath, instantTime, fileID)).thenReturn(expected);
actual = fsView.getBaseFileOn(partitionPath, instantTime, fileID);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getBaseFileOn(partitionPath, instantTime, fileID))
.thenThrow(new RuntimeException());
when(secondary.getBaseFileOn(partitionPath, instantTime, fileID)).thenReturn(expected);
@@ -297,10 +300,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getLatestBaseFilesInRange(commitsToReturn)).thenReturn(testBaseFileStream);
actual = fsView.getLatestBaseFilesInRange(commitsToReturn);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestBaseFilesInRange(commitsToReturn)).thenThrow(new RuntimeException());
when(secondary.getLatestBaseFilesInRange(commitsToReturn)).thenReturn(testBaseFileStream);
actual = fsView.getLatestBaseFilesInRange(commitsToReturn);
@@ -327,10 +330,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getAllBaseFiles(partitionPath)).thenReturn(testBaseFileStream);
actual = fsView.getAllBaseFiles(partitionPath);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getAllBaseFiles(partitionPath)).thenThrow(new RuntimeException());
when(secondary.getAllBaseFiles(partitionPath)).thenReturn(testBaseFileStream);
actual = fsView.getAllBaseFiles(partitionPath);
@@ -357,10 +360,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getLatestFileSlices(partitionPath)).thenReturn(testFileSliceStream);
actual = fsView.getLatestFileSlices(partitionPath);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestFileSlices(partitionPath)).thenThrow(new RuntimeException());
when(secondary.getLatestFileSlices(partitionPath)).thenReturn(testFileSliceStream);
actual = fsView.getLatestFileSlices(partitionPath);
@@ -387,10 +390,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getLatestFileSlicesIncludingInflight(partitionPath)).thenReturn(testFileSliceStream);
actual = fsView.getLatestFileSlicesIncludingInflight(partitionPath);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestFileSlicesIncludingInflight(partitionPath)).thenThrow(new RuntimeException());
when(secondary.getLatestFileSlicesIncludingInflight(partitionPath)).thenReturn(testFileSliceStream);
actual = fsView.getLatestFileSlicesIncludingInflight(partitionPath);
@@ -415,10 +418,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getLatestUnCompactedFileSlices(partitionPath)).thenReturn(testFileSliceStream);
actual = fsView.getLatestUnCompactedFileSlices(partitionPath);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestUnCompactedFileSlices(partitionPath)).thenThrow(new RuntimeException());
when(secondary.getLatestUnCompactedFileSlices(partitionPath)).thenReturn(testFileSliceStream);
actual = fsView.getLatestUnCompactedFileSlices(partitionPath);
@@ -447,10 +450,10 @@ public class TestPriorityBasedFileSystemView {
.thenReturn(testFileSliceStream);
actual = fsView.getLatestFileSlicesBeforeOrOn(partitionPath, maxCommitTime, false);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestFileSlicesBeforeOrOn(partitionPath, maxCommitTime, false))
.thenThrow(new RuntimeException());
when(secondary.getLatestFileSlicesBeforeOrOn(partitionPath, maxCommitTime, false))
@@ -483,10 +486,10 @@ public class TestPriorityBasedFileSystemView {
.thenReturn(testFileSliceStream);
actual = fsView.getLatestMergedFileSlicesBeforeOrOn(partitionPath, maxInstantTime);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestMergedFileSlicesBeforeOrOn(partitionPath, maxInstantTime))
.thenThrow(new RuntimeException());
when(secondary.getLatestMergedFileSlicesBeforeOrOn(partitionPath, maxInstantTime))
@@ -517,10 +520,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getLatestFileSliceInRange(commitsToReturn)).thenReturn(testFileSliceStream);
actual = fsView.getLatestFileSliceInRange(commitsToReturn);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestFileSliceInRange(commitsToReturn)).thenThrow(new RuntimeException());
when(secondary.getLatestFileSliceInRange(commitsToReturn)).thenReturn(testFileSliceStream);
actual = fsView.getLatestFileSliceInRange(commitsToReturn);
@@ -547,10 +550,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getAllFileSlices(partitionPath)).thenReturn(testFileSliceStream);
actual = fsView.getAllFileSlices(partitionPath);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getAllFileSlices(partitionPath)).thenThrow(new RuntimeException());
when(secondary.getAllFileSlices(partitionPath)).thenReturn(testFileSliceStream);
actual = fsView.getAllFileSlices(partitionPath);
@@ -579,10 +582,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getAllFileGroups(partitionPath)).thenReturn(expected);
actual = fsView.getAllFileGroups(partitionPath);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getAllFileGroups(partitionPath)).thenThrow(new RuntimeException());
when(secondary.getAllFileGroups(partitionPath)).thenReturn(expected);
actual = fsView.getAllFileGroups(partitionPath);
@@ -610,10 +613,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getPendingCompactionOperations()).thenReturn(expected);
actual = fsView.getPendingCompactionOperations();
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getPendingCompactionOperations()).thenThrow(new RuntimeException());
when(secondary.getPendingCompactionOperations()).thenReturn(expected);
actual = fsView.getPendingCompactionOperations();
@@ -641,10 +644,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getPendingLogCompactionOperations()).thenReturn(expected);
actual = fsView.getPendingLogCompactionOperations();
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getPendingLogCompactionOperations()).thenThrow(new RuntimeException());
when(secondary.getPendingLogCompactionOperations()).thenReturn(expected);
actual = fsView.getPendingLogCompactionOperations();
@@ -672,7 +675,7 @@ public class TestPriorityBasedFileSystemView {
@Test
public void testClose_withSecondaryInitialized() {
// force secondary view to initialize
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
fsView.getSecondaryView();
fsView.close();
verify(primary, times(1)).close();
@@ -688,7 +691,7 @@ public class TestPriorityBasedFileSystemView {
@Test
public void testReset_withSecondaryInitialized() {
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
fsView.getSecondaryView();
fsView.reset();
verify(primary, times(1)).reset();
@@ -703,10 +706,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getLastInstant()).thenReturn(expected);
actual = fsView.getLastInstant();
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLastInstant()).thenThrow(new RuntimeException());
when(secondary.getLastInstant()).thenReturn(expected);
actual = fsView.getLastInstant();
@@ -732,10 +735,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getTimeline()).thenReturn(expected);
actual = fsView.getTimeline();
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getTimeline()).thenThrow(new RuntimeException());
when(secondary.getTimeline()).thenReturn(expected);
actual = fsView.getTimeline();
@@ -762,7 +765,7 @@ public class TestPriorityBasedFileSystemView {
@Test
public void testSync_withSecondaryInitialized() {
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
fsView.getSecondaryView();
fsView.sync();
verify(primary, times(1)).sync();
@@ -779,10 +782,10 @@ public class TestPriorityBasedFileSystemView {
when(primary.getLatestFileSlice(partitionPath, fileID)).thenReturn(expected);
actual = fsView.getLatestFileSlice(partitionPath, fileID);
assertEquals(expected, actual);
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
when(primary.getLatestFileSlice(partitionPath, fileID)).thenThrow(new RuntimeException());
when(secondary.getLatestFileSlice(partitionPath, fileID)).thenReturn(expected);
actual = fsView.getLatestFileSlice(partitionPath, fileID);
@@ -807,10 +810,10 @@ public class TestPriorityBasedFileSystemView {
fsView.loadPartitions(Collections.singletonList(partitionPath));
verify(primary, times(1)).loadPartitions(Collections.singletonList(partitionPath));
verify(secondary, never()).loadPartitions(any());
- verify(secondaryViewSupplier, never()).get();
+ verify(secondaryViewCreator, never()).apply(engineContext);
resetMocks();
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
doThrow(new RuntimeException()).when(primary).loadPartitions(Collections.singletonList(partitionPath));
fsView.loadPartitions(Collections.singletonList(partitionPath));
verify(primary, times(1)).loadPartitions(Collections.singletonList(partitionPath));
@@ -830,7 +833,7 @@ public class TestPriorityBasedFileSystemView {
@Test
public void testGetSecondaryView() {
- when(secondaryViewSupplier.get()).thenReturn(secondary);
+ when(secondaryViewCreator.apply(engineContext)).thenReturn(secondary);
assertEquals(secondary, fsView.getSecondaryView());
}
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FormatUtils.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FormatUtils.java
index 0628673b8e5..7c7d9bd7dec 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FormatUtils.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FormatUtils.java
@@ -24,6 +24,7 @@ import org.apache.hudi.common.engine.EngineType;
import org.apache.hudi.common.model.HoodieOperation;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordMerger;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner;
import org.apache.hudi.common.table.log.HoodieUnMergedLogRecordScanner;
import org.apache.hudi.common.util.DefaultSizeEstimator;
@@ -139,6 +140,7 @@ public class FormatUtils {
new DefaultSizeEstimator<>(),
new DefaultSizeEstimator<>(),
writeConfig.getCommonConfig().getSpillableDiskMapType(),
+ new DefaultSerializer<>(),
writeConfig.getCommonConfig().isBitCaskDiskMapCompressionEnabled());
} catch (IOException e) {
throw new HoodieIOException(
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestCustomSerDe.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestCustomSerDe.java
index c6fa0e7108d..4b5ce9dc081 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestCustomSerDe.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestCustomSerDe.java
@@ -22,6 +22,7 @@ import org.apache.hudi.common.model.EventTimeAvroPayload;
import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.util.collection.BitCaskDiskMap;
import org.apache.hudi.common.util.collection.RocksDbDiskMap;
@@ -54,18 +55,20 @@ public class TestCustomSerDe extends HoodieCommonTestHarness {
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testBitCaskDiskMapPutDecimal(boolean isCompressionEnabled) throws IOException {
- BitCaskDiskMap<String, HoodieRecord> bitCaskDiskMap = new BitCaskDiskMap<>(basePath, isCompressionEnabled);
- HoodieRecord avroRecord = createAvroRecordWithDecimalOrderingField();
- bitCaskDiskMap.put(avroRecord.getRecordKey(), avroRecord);
- assertDoesNotThrow(() -> bitCaskDiskMap.get(avroRecord.getRecordKey()));
+ try (BitCaskDiskMap<String, HoodieRecord> bitCaskDiskMap = new BitCaskDiskMap<>(basePath, new DefaultSerializer<>(), isCompressionEnabled)) {
+ HoodieRecord avroRecord = createAvroRecordWithDecimalOrderingField();
+ bitCaskDiskMap.put(avroRecord.getRecordKey(), avroRecord);
+ assertDoesNotThrow(() -> bitCaskDiskMap.get(avroRecord.getRecordKey()));
+ }
}
@Test
public void testRocksDbDiskMapPutDecimal() throws IOException {
- RocksDbDiskMap<String, HoodieRecord> rocksDbBasedMap = new RocksDbDiskMap<>(basePath);
- HoodieRecord avroRecord = createAvroRecordWithDecimalOrderingField();
- rocksDbBasedMap.put(avroRecord.getRecordKey(), avroRecord);
- assertDoesNotThrow(() -> rocksDbBasedMap.get(avroRecord.getRecordKey()));
+ try (RocksDbDiskMap<String, HoodieRecord> rocksDbBasedMap = new RocksDbDiskMap<>(basePath, new DefaultSerializer<>())) {
+ HoodieRecord avroRecord = createAvroRecordWithDecimalOrderingField();
+ rocksDbBasedMap.put(avroRecord.getRecordKey(), avroRecord);
+ assertDoesNotThrow(() -> rocksDbBasedMap.get(avroRecord.getRecordKey()));
+ }
}
private static HoodieRecord createAvroRecordWithDecimalOrderingField() {
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDBBasedIncrementalFSViewSync.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDBBasedIncrementalFSViewSync.java
index 082277e71c9..86cc6752c24 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDBBasedIncrementalFSViewSync.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDBBasedIncrementalFSViewSync.java
@@ -20,6 +20,8 @@ package org.apache.hudi.common.table.view;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import java.io.IOException;
import java.nio.file.Files;
@@ -33,7 +35,9 @@ public class TestRocksDBBasedIncrementalFSViewSync extends TestIncrementalFSView
protected SyncableFileSystemView getFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline timeline)
throws IOException {
String subdirPath = Files.createTempDirectory(tempDir, null).toAbsolutePath().toString();
- return new RocksDbBasedFileSystemView(metaClient, timeline, FileSystemViewStorageConfig.newBuilder()
+ HoodieTableMetadata tableMetadata = new FileSystemBackedTableMetadata(getEngineContext(), metaClient.getTableConfig(), metaClient.getStorage(),
+ metaClient.getBasePath().toString());
+ return new RocksDbBasedFileSystemView(tableMetadata, metaClient, timeline, FileSystemViewStorageConfig.newBuilder()
.withRocksDBPath(subdirPath).withIncrementalTimelineSync(true).build());
}
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java
index 7793178d102..4079bef8179 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java
@@ -19,6 +19,8 @@
package org.apache.hudi.common.table.view;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import java.io.IOException;
import java.nio.file.Files;
@@ -31,7 +33,9 @@ public class TestRocksDbBasedFileSystemView extends TestHoodieTableFileSystemVie
@Override
protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException {
String subdirPath = Files.createTempDirectory(tempDir, null).toAbsolutePath().toString();
- return new RocksDbBasedFileSystemView(metaClient, timeline,
+ HoodieTableMetadata tableMetadata = new FileSystemBackedTableMetadata(getEngineContext(), metaClient.getTableConfig(), metaClient.getStorage(),
+ metaClient.getBasePath().toString());
+ return new RocksDbBasedFileSystemView(tableMetadata, metaClient, timeline,
FileSystemViewStorageConfig.newBuilder().withRocksDBPath(subdirPath).build());
}
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedFileSystemView.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedFileSystemView.java
index 8109249c19d..5f2c1645f45 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedFileSystemView.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedFileSystemView.java
@@ -20,6 +20,8 @@ package org.apache.hudi.common.table.view;
import org.apache.hudi.common.config.HoodieCommonConfig;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadata;
/**
* Tests spillable map based file system view {@link SpillableMapBasedFileSystemView}.
@@ -28,7 +30,9 @@ public class TestSpillableMapBasedFileSystemView extends TestHoodieTableFileSyst
@Override
protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) {
- return new SpillableMapBasedFileSystemView(metaClient, timeline, FileSystemViewStorageConfig.newBuilder()
+ HoodieTableMetadata tableMetadata = new FileSystemBackedTableMetadata(getEngineContext(), metaClient.getTableConfig(), metaClient.getStorage(),
+ metaClient.getBasePath().toString());
+ return new SpillableMapBasedFileSystemView(tableMetadata, metaClient, timeline, FileSystemViewStorageConfig.newBuilder()
// pure disk base View
.withStorageType(FileSystemViewStorageType.SPILLABLE_DISK).withMaxMemoryForView(0L).build(),
HoodieCommonConfig.newBuilder().build());
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedIncrementalFSViewSync.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedIncrementalFSViewSync.java
index c678dd2e48c..0d97691be08 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedIncrementalFSViewSync.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedIncrementalFSViewSync.java
@@ -21,6 +21,8 @@ package org.apache.hudi.common.table.view;
import org.apache.hudi.common.config.HoodieCommonConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadata;
/**
* Tests spillable map based incremental fs view sync {@link SpillableMapBasedFileSystemView}.
@@ -29,7 +31,9 @@ public class TestSpillableMapBasedIncrementalFSViewSync extends TestIncrementalF
@Override
protected SyncableFileSystemView getFileSystemView(HoodieTableMetaClient metaClient, HoodieTimeline timeline) {
- return new SpillableMapBasedFileSystemView(metaClient, timeline,
+ HoodieTableMetadata tableMetadata = new FileSystemBackedTableMetadata(getEngineContext(), metaClient.getTableConfig(), metaClient.getStorage(),
+ metaClient.getBasePath().toString());
+ return new SpillableMapBasedFileSystemView(tableMetadata, metaClient, timeline,
FileSystemViewStorageConfig.newBuilder().withMaxMemoryForView(0L).withIncrementalTimelineSync(true).build(),
HoodieCommonConfig.newBuilder().build());
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
index 590a7818c91..ad586fc4a4a 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
@@ -19,6 +19,8 @@
package org.apache.hudi.common.testutils;
import org.apache.hudi.common.config.HoodieReaderConfig;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieLogFile;
@@ -46,6 +48,7 @@ import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
@@ -89,6 +92,7 @@ public class HoodieCommonTestHarness {
protected URI baseUri;
protected HoodieTestDataGenerator dataGen;
protected HoodieTableMetaClient metaClient;
+ private HoodieEngineContext engineContext;
@TempDir
public java.nio.file.Path tempDir;
@@ -178,7 +182,7 @@ public class HoodieCommonTestHarness {
}
protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline, boolean enableIncrementalTimelineSync) {
- return new HoodieTableFileSystemView(metaClient, timeline, enableIncrementalTimelineSync);
+ return HoodieTableFileSystemView.fileListingBasedFileSystemView(getEngineContext(), metaClient, timeline, enableIncrementalTimelineSync);
}
protected SyncableFileSystemView getFileSystemView(HoodieTableMetaClient metaClient) throws IOException {
@@ -281,6 +285,13 @@ public class HoodieCommonTestHarness {
return !pendingInstants.isEmpty();
}
+ protected HoodieEngineContext getEngineContext() {
+ if (engineContext == null) {
+ this.engineContext = new HoodieLocalEngineContext(new HadoopStorageConfiguration(false));
+ }
+ return this.engineContext;
+ }
+
protected static List<HoodieLogFile> writeLogFiles(StoragePath partitionPath,
Schema schema,
List<HoodieRecord> records,
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java
index 68b612d90c1..48b2e5d39ec 100755
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java
@@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.testutils.AvroBinaryTestPayload;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.InProcessTimeGenerator;
@@ -74,65 +75,67 @@ public class TestBitCaskDiskMap extends HoodieCommonTestHarness {
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testSimpleInsert(boolean isCompressionEnabled) throws IOException, URISyntaxException {
- BitCaskDiskMap records = new BitCaskDiskMap<>(basePath, isCompressionEnabled);
- SchemaTestUtil testUtil = new SchemaTestUtil();
- List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
- List<String> recordKeys = SpillableMapTestUtils.upsertRecords(iRecords, records);
-
- Map<String, IndexedRecord> originalRecords = iRecords.stream()
- .collect(Collectors.toMap(k -> ((GenericRecord) k).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString(), v -> v));
-
- // make sure records have spilled to disk
- assertTrue(records.sizeOfFileOnDiskInBytes() > 0);
- Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = records.iterator();
- while (itr.hasNext()) {
- HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
- assert recordKeys.contains(rec.getRecordKey());
- IndexedRecord originalRecord = originalRecords.get(rec.getRecordKey());
- HoodieAvroPayload payload = (HoodieAvroPayload) rec.getData();
- Option<IndexedRecord> value = payload.getInsertValue(HoodieAvroUtils.addMetadataFields(getSimpleSchema()));
- assertEquals(originalRecord, value.get());
- }
+ try (BitCaskDiskMap records = new BitCaskDiskMap<>(basePath, new DefaultSerializer<>(), isCompressionEnabled)) {
+ SchemaTestUtil testUtil = new SchemaTestUtil();
+ List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
+ List<String> recordKeys = SpillableMapTestUtils.upsertRecords(iRecords, records);
+
+ Map<String, IndexedRecord> originalRecords = iRecords.stream()
+ .collect(Collectors.toMap(k -> ((GenericRecord) k).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString(), v -> v));
+
+ // make sure records have spilled to disk
+ assertTrue(records.sizeOfFileOnDiskInBytes() > 0);
+ Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = records.iterator();
+ while (itr.hasNext()) {
+ HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
+ assert recordKeys.contains(rec.getRecordKey());
+ IndexedRecord originalRecord = originalRecords.get(rec.getRecordKey());
+ HoodieAvroPayload payload = (HoodieAvroPayload) rec.getData();
+ Option<IndexedRecord> value = payload.getInsertValue(HoodieAvroUtils.addMetadataFields(getSimpleSchema()));
+ assertEquals(originalRecord, value.get());
+ }
- verifyCleanup(records);
+ verifyCleanup(records);
+ }
}
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testSimpleInsertWithoutHoodieMetadata(boolean isCompressionEnabled) throws IOException, URISyntaxException {
- BitCaskDiskMap records = new BitCaskDiskMap<>(basePath, isCompressionEnabled);
- SchemaTestUtil testUtil = new SchemaTestUtil();
- List<HoodieRecord> hoodieRecords = testUtil.generateHoodieTestRecordsWithoutHoodieMetadata(0, 1000);
- Set<String> recordKeys = new HashSet<>();
- // insert generated records into the map
- hoodieRecords.forEach(r -> {
- records.put(r.getRecordKey(), r);
- recordKeys.add(r.getRecordKey());
- });
- // make sure records have spilled to disk
- assertTrue(records.sizeOfFileOnDiskInBytes() > 0);
- Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = records.iterator();
- List<HoodieRecord> oRecords = new ArrayList<>();
- while (itr.hasNext()) {
- HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
- oRecords.add(rec);
- assert recordKeys.contains(rec.getRecordKey());
- }
+ try (BitCaskDiskMap records = new BitCaskDiskMap<>(basePath, new DefaultSerializer<>(), isCompressionEnabled)) {
+ SchemaTestUtil testUtil = new SchemaTestUtil();
+ List<HoodieRecord> hoodieRecords = testUtil.generateHoodieTestRecordsWithoutHoodieMetadata(0, 1000);
+ Set<String> recordKeys = new HashSet<>();
+ // insert generated records into the map
+ hoodieRecords.forEach(r -> {
+ records.put(r.getRecordKey(), r);
+ recordKeys.add(r.getRecordKey());
+ });
+ // make sure records have spilled to disk
+ assertTrue(records.sizeOfFileOnDiskInBytes() > 0);
+ Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = records.iterator();
+ List<HoodieRecord> oRecords = new ArrayList<>();
+ while (itr.hasNext()) {
+ HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
+ oRecords.add(rec);
+ assert recordKeys.contains(rec.getRecordKey());
+ }
- // test iterator with predicate
- String firstKey = recordKeys.stream().findFirst().get();
- recordKeys.remove(firstKey);
- itr = records.iterator(key -> !key.equals(firstKey));
- int cntSize = 0;
- while (itr.hasNext()) {
- HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
- cntSize++;
- assert recordKeys.contains(rec.getRecordKey());
- }
- assertEquals(recordKeys.size(), cntSize);
+ // test iterator with predicate
+ String firstKey = recordKeys.stream().findFirst().get();
+ recordKeys.remove(firstKey);
+ itr = records.iterator(key -> !key.equals(firstKey));
+ int cntSize = 0;
+ while (itr.hasNext()) {
+ HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
+ cntSize++;
+ assert recordKeys.contains(rec.getRecordKey());
+ }
+ assertEquals(recordKeys.size(), cntSize);
- verifyCleanup(records);
+ verifyCleanup(records);
+ }
}
@ParameterizedTest
@@ -140,44 +143,45 @@ public class TestBitCaskDiskMap extends HoodieCommonTestHarness {
public void testSimpleUpsert(boolean isCompressionEnabled) throws IOException, URISyntaxException {
Schema schema = HoodieAvroUtils.addMetadataFields(getSimpleSchema());
- BitCaskDiskMap records = new BitCaskDiskMap<>(basePath, isCompressionEnabled);
- SchemaTestUtil testUtil = new SchemaTestUtil();
- List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
-
- // perform some inserts
- List<String> recordKeys = SpillableMapTestUtils.upsertRecords(iRecords, records);
-
- long fileSize = records.sizeOfFileOnDiskInBytes();
- // make sure records have spilled to disk
- assertTrue(fileSize > 0);
-
- // generate updates from inserts
- List<IndexedRecord> updatedRecords = SchemaTestUtil.updateHoodieTestRecords(recordKeys,
- testUtil.generateHoodieTestRecords(0, 100), InProcessTimeGenerator.createNewInstantTime());
- String newCommitTime =
- ((GenericRecord) updatedRecords.get(0)).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
-
- // perform upserts
- recordKeys = SpillableMapTestUtils.upsertRecords(updatedRecords, records);
-
- // upserts should be appended to the existing file, hence increasing the sizeOfFile on disk
- assertTrue(records.sizeOfFileOnDiskInBytes() > fileSize);
-
- // Upserted records (on disk) should have the latest commit time
- Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = records.iterator();
- while (itr.hasNext()) {
- HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
- assert recordKeys.contains(rec.getRecordKey());
- try {
- IndexedRecord indexedRecord = (IndexedRecord) rec.getData().getInsertValue(schema).get();
- String latestCommitTime =
- ((GenericRecord) indexedRecord).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
- assertEquals(latestCommitTime, newCommitTime);
- } catch (IOException io) {
- throw new UncheckedIOException(io);
+ try (BitCaskDiskMap records = new BitCaskDiskMap<>(basePath, new DefaultSerializer<>(), isCompressionEnabled)) {
+ SchemaTestUtil testUtil = new SchemaTestUtil();
+ List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
+
+ // perform some inserts
+ List<String> recordKeys = SpillableMapTestUtils.upsertRecords(iRecords, records);
+
+ long fileSize = records.sizeOfFileOnDiskInBytes();
+ // make sure records have spilled to disk
+ assertTrue(fileSize > 0);
+
+ // generate updates from inserts
+ List<IndexedRecord> updatedRecords = SchemaTestUtil.updateHoodieTestRecords(recordKeys,
+ testUtil.generateHoodieTestRecords(0, 100), InProcessTimeGenerator.createNewInstantTime());
+ String newCommitTime =
+ ((GenericRecord) updatedRecords.get(0)).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
+
+ // perform upserts
+ recordKeys = SpillableMapTestUtils.upsertRecords(updatedRecords, records);
+
+ // upserts should be appended to the existing file, hence increasing the sizeOfFile on disk
+ assertTrue(records.sizeOfFileOnDiskInBytes() > fileSize);
+
+ // Upserted records (on disk) should have the latest commit time
+ Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = records.iterator();
+ while (itr.hasNext()) {
+ HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
+ assert recordKeys.contains(rec.getRecordKey());
+ try {
+ IndexedRecord indexedRecord = (IndexedRecord) rec.getData().getInsertValue(schema).get();
+ String latestCommitTime =
+ ((GenericRecord) indexedRecord).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
+ assertEquals(latestCommitTime, newCommitTime);
+ } catch (IOException io) {
+ throw new UncheckedIOException(io);
+ }
}
+ verifyCleanup(records);
}
- verifyCleanup(records);
}
@Test
@@ -224,24 +228,25 @@ public class TestBitCaskDiskMap extends HoodieCommonTestHarness {
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testPutAll(boolean isCompressionEnabled) throws IOException, URISyntaxException {
- BitCaskDiskMap<String, HoodieRecord> records = new BitCaskDiskMap<>(basePath, isCompressionEnabled);
- SchemaTestUtil testUtil = new SchemaTestUtil();
- List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
- Map<String, HoodieRecord> recordMap = new HashMap<>();
- iRecords.forEach(r -> {
- String key = ((GenericRecord) r).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
- String partitionPath = ((GenericRecord) r).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString();
- HoodieRecord value = new HoodieAvroRecord<>(new HoodieKey(key, partitionPath), new HoodieAvroPayload(Option.of((GenericRecord) r)));
- recordMap.put(key, value);
- });
-
- records.putAll(recordMap);
- // make sure records have spilled to disk
- assertTrue(records.sizeOfFileOnDiskInBytes() > 0);
-
- // make sure all added records are present
- for (Map.Entry<String, HoodieRecord> entry : records.entrySet()) {
- assertTrue(recordMap.containsKey(entry.getKey()));
+ try (BitCaskDiskMap<String, HoodieRecord> records = new BitCaskDiskMap<>(basePath, new DefaultSerializer<>(), isCompressionEnabled)) {
+ SchemaTestUtil testUtil = new SchemaTestUtil();
+ List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
+ Map<String, HoodieRecord> recordMap = new HashMap<>();
+ iRecords.forEach(r -> {
+ String key = ((GenericRecord) r).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
+ String partitionPath = ((GenericRecord) r).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString();
+ HoodieRecord value = new HoodieAvroRecord<>(new HoodieKey(key, partitionPath), new HoodieAvroPayload(Option.of((GenericRecord) r)));
+ recordMap.put(key, value);
+ });
+
+ records.putAll(recordMap);
+ // make sure records have spilled to disk
+ assertTrue(records.sizeOfFileOnDiskInBytes() > 0);
+
+ // make sure all added records are present
+ for (Map.Entry<String, HoodieRecord> entry : records.entrySet()) {
+ assertTrue(recordMap.containsKey(entry.getKey()));
+ }
}
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java
index 9d7f29a4fe9..cc1c0fe440d 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java
@@ -25,6 +25,7 @@ import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;
import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.InProcessTimeGenerator;
import org.apache.hudi.common.testutils.SchemaTestUtil;
@@ -84,7 +85,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
try (ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records =
new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(),
- new HoodieRecordSizeEstimator(schema), diskMapType, isCompressionEnabled)) { // 16B
+ new HoodieRecordSizeEstimator(schema), diskMapType, new DefaultSerializer<>(), isCompressionEnabled)) { // 16B
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
@@ -119,7 +120,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
try (ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records =
new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(),
- new HoodieRecordSizeEstimator(schema), diskMapType, isCompressionEnabled)) { // 16B
+ new HoodieRecordSizeEstimator(schema), diskMapType, new DefaultSerializer<>(), isCompressionEnabled)) { // 16B
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
@@ -161,7 +162,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
try (ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records =
new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(),
- new HoodieRecordSizeEstimator(schema), diskMapType, isCompressionEnabled)) { // 16B
+ new HoodieRecordSizeEstimator(schema), diskMapType, new DefaultSerializer<>(), isCompressionEnabled)) { // 16B
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
@@ -219,7 +220,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
try (ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records = new ExternalSpillableMap<>(16L,
failureOutputPath, new DefaultSizeEstimator(),
- new HoodieRecordSizeEstimator(schema), diskMapType, isCompressionEnabled)) { // 16B
+ new HoodieRecordSizeEstimator(schema), diskMapType, new DefaultSerializer<>(), isCompressionEnabled)) { // 16B
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
@@ -244,7 +245,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
try (ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records =
new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(),
- new HoodieRecordSizeEstimator(schema), diskMapType, isCompressionEnabled)) { // 16B
+ new HoodieRecordSizeEstimator(schema), diskMapType, new DefaultSerializer<>(), isCompressionEnabled)) { // 16B
List<String> recordKeys = new ArrayList<>();
// Ensure we spill to disk
@@ -301,7 +302,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
try (ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records =
new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(),
- new HoodieRecordSizeEstimator(schema), diskMapType, isCompressionEnabled)) { // 16B
+ new HoodieRecordSizeEstimator(schema), diskMapType, new DefaultSerializer<>(), isCompressionEnabled)) { // 16B
SchemaTestUtil testUtil = new SchemaTestUtil();
List<String> recordKeys = new ArrayList<>();
@@ -369,7 +370,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
try (ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records =
new ExternalSpillableMap<>(16L, basePath, new DefaultSizeEstimator(),
- new HoodieRecordSizeEstimator(schema), diskMapType, isCompressionEnabled)) {
+ new HoodieRecordSizeEstimator(schema), diskMapType, new DefaultSerializer<>(), isCompressionEnabled)) {
List<String> recordKeys = new ArrayList<>();
SchemaTestUtil testUtil = new SchemaTestUtil();
@@ -418,7 +419,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
long totalEstimatedSizeWith100Records = (long) ((estimatedPayloadSize * 100) / 0.8);
try (ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records =
new ExternalSpillableMap<>(totalEstimatedSizeWith100Records, basePath, new DefaultSizeEstimator(),
- new HoodieRecordSizeEstimator(schema), diskMapType, isCompressionEnabled)) {
+ new HoodieRecordSizeEstimator(schema), diskMapType, new DefaultSerializer<>(), isCompressionEnabled)) {
// Insert 100 records and then in-memory map will contain 100 records.
SpillableMapTestUtils.upsertRecords(iRecords, records);
@@ -447,7 +448,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
// validate that operations on an empty map work as expected
try (ExternalSpillableMap<String, HoodieRecord<? extends HoodieRecordPayload>> records =
new ExternalSpillableMap<>(10, basePath, new DefaultSizeEstimator(),
- new DefaultSizeEstimator<>(), diskMapType, false)) {
+ new DefaultSizeEstimator<>(), diskMapType, new DefaultSerializer<>(),false)) {
assertTrue(records.isEmpty());
assertFalse(records.containsKey("key"));
assertFalse(records.containsValue("value"));
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java
index 96a106ec901..f45512aa8c4 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java
@@ -20,6 +20,7 @@ package org.apache.hudi.common.util.collection;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.SchemaTestUtil;
import org.apache.hudi.common.testutils.SpillableMapTestUtils;
@@ -49,20 +50,21 @@ public class TestRocksDbBasedMap extends HoodieCommonTestHarness {
@Test
public void testSimple() throws IOException, URISyntaxException {
- RocksDbDiskMap records = new RocksDbDiskMap(basePath);
- SchemaTestUtil testUtil = new SchemaTestUtil();
- List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
- ((GenericRecord) iRecords.get(0)).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
- List<String> recordKeys = SpillableMapTestUtils.upsertRecords(iRecords, records);
+ try (RocksDbDiskMap records = new RocksDbDiskMap(basePath, new DefaultSerializer<>())) {
+ SchemaTestUtil testUtil = new SchemaTestUtil();
+ List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
+ ((GenericRecord) iRecords.get(0)).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
+ List<String> recordKeys = SpillableMapTestUtils.upsertRecords(iRecords, records);
- // make sure records have spilled to disk
- Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = records.iterator();
- List<HoodieRecord> oRecords = new ArrayList<>();
- while (itr.hasNext()) {
- HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
- oRecords.add(rec);
- assert recordKeys.contains(rec.getRecordKey());
+ // make sure records have spilled to disk
+ Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = records.iterator();
+ List<HoodieRecord> oRecords = new ArrayList<>();
+ while (itr.hasNext()) {
+ HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
+ oRecords.add(rec);
+ assert recordKeys.contains(rec.getRecordKey());
+ }
+ assertEquals(recordKeys.size(), oRecords.size());
}
- assertEquals(recordKeys.size(), oRecords.size());
}
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbDiskMap.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbDiskMap.java
index a013c9e638c..fcca63beb8e 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbDiskMap.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbDiskMap.java
@@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.InProcessTimeGenerator;
import org.apache.hudi.common.testutils.SchemaTestUtil;
@@ -65,145 +66,151 @@ public class TestRocksDbDiskMap extends HoodieCommonTestHarness {
@Test
public void testSimpleInsertSequential() throws IOException, URISyntaxException {
- RocksDbDiskMap<String, HoodieRecord<? extends HoodieRecordPayload>> rocksDBBasedMap = new RocksDbDiskMap<>(basePath);
- List<String> recordKeys = setupMapWithRecords(rocksDBBasedMap, 100);
-
- Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = rocksDBBasedMap.iterator();
- int cntSize = 0;
- while (itr.hasNext()) {
- HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
- cntSize++;
- assert recordKeys.contains(rec.getRecordKey());
- }
- assertEquals(recordKeys.size(), cntSize);
-
- // Test value stream
- long currentTimeMs = System.currentTimeMillis();
- List<HoodieRecord<? extends HoodieRecordPayload>> values =
- rocksDBBasedMap.valueStream().collect(Collectors.toList());
- cntSize = 0;
- for (HoodieRecord value : values) {
- assert recordKeys.contains(value.getRecordKey());
- cntSize++;
+ try (RocksDbDiskMap<String, HoodieRecord<? extends HoodieRecordPayload>> rocksDBBasedMap = new RocksDbDiskMap<>(basePath, new DefaultSerializer<>())) {
+ List<String> recordKeys = setupMapWithRecords(rocksDBBasedMap, 100);
+
+ Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = rocksDBBasedMap.iterator();
+ int cntSize = 0;
+ while (itr.hasNext()) {
+ HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
+ cntSize++;
+ assert recordKeys.contains(rec.getRecordKey());
+ }
+ assertEquals(recordKeys.size(), cntSize);
+
+ // Test value stream
+ long currentTimeMs = System.currentTimeMillis();
+ List<HoodieRecord<? extends HoodieRecordPayload>> values =
+ rocksDBBasedMap.valueStream().collect(Collectors.toList());
+ cntSize = 0;
+ for (HoodieRecord value : values) {
+ assert recordKeys.contains(value.getRecordKey());
+ cntSize++;
+ }
+ assertEquals(recordKeys.size(), cntSize);
}
- assertEquals(recordKeys.size(), cntSize);
}
@Test
public void testSimpleInsertRandomAccess() throws IOException, URISyntaxException {
- RocksDbDiskMap rocksDBBasedMap = new RocksDbDiskMap<>(basePath);
- List<String> recordKeys = setupMapWithRecords(rocksDBBasedMap, 100);
+ try (RocksDbDiskMap rocksDBBasedMap = new RocksDbDiskMap<>(basePath, new DefaultSerializer<>())) {
+ List<String> recordKeys = setupMapWithRecords(rocksDBBasedMap, 100);
- Random random = new Random();
- for (int i = 0; i < recordKeys.size(); i++) {
- String key = recordKeys.get(random.nextInt(recordKeys.size()));
- assert rocksDBBasedMap.get(key) != null;
+ Random random = new Random();
+ for (int i = 0; i < recordKeys.size(); i++) {
+ String key = recordKeys.get(random.nextInt(recordKeys.size()));
+ assert rocksDBBasedMap.get(key) != null;
+ }
}
}
@Test
public void testSimpleInsertWithoutHoodieMetadata() throws IOException, URISyntaxException {
- RocksDbDiskMap rocksDBBasedMap = new RocksDbDiskMap<>(basePath);
- SchemaTestUtil testUtil = new SchemaTestUtil();
- List<HoodieRecord> hoodieRecords = testUtil.generateHoodieTestRecordsWithoutHoodieMetadata(0, 1000);
- Set<String> recordKeys = new HashSet<>();
- // insert generated records into the map
- hoodieRecords.forEach(r -> {
- rocksDBBasedMap.put(r.getRecordKey(), r);
- recordKeys.add(r.getRecordKey());
- });
- // make sure records have spilled to disk
- assertTrue(rocksDBBasedMap.sizeOfFileOnDiskInBytes() > 0);
- Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = rocksDBBasedMap.iterator();
- int cntSize = 0;
- while (itr.hasNext()) {
- HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
- cntSize++;
- assert recordKeys.contains(rec.getRecordKey());
- }
- assertEquals(recordKeys.size(), cntSize);
-
- // test iterator with predicate
- String firstKey = recordKeys.stream().findFirst().get();
- recordKeys.remove(firstKey);
- itr = rocksDBBasedMap.iterator(key -> !key.equals(firstKey));
- cntSize = 0;
- while (itr.hasNext()) {
- HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
- cntSize++;
- assert recordKeys.contains(rec.getRecordKey());
+ try (RocksDbDiskMap rocksDBBasedMap = new RocksDbDiskMap<>(basePath, new DefaultSerializer<>())) {
+ SchemaTestUtil testUtil = new SchemaTestUtil();
+ List<HoodieRecord> hoodieRecords = testUtil.generateHoodieTestRecordsWithoutHoodieMetadata(0, 1000);
+ Set<String> recordKeys = new HashSet<>();
+ // insert generated records into the map
+ hoodieRecords.forEach(r -> {
+ rocksDBBasedMap.put(r.getRecordKey(), r);
+ recordKeys.add(r.getRecordKey());
+ });
+ // make sure records have spilled to disk
+ assertTrue(rocksDBBasedMap.sizeOfFileOnDiskInBytes() > 0);
+ Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = rocksDBBasedMap.iterator();
+ int cntSize = 0;
+ while (itr.hasNext()) {
+ HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
+ cntSize++;
+ assert recordKeys.contains(rec.getRecordKey());
+ }
+ assertEquals(recordKeys.size(), cntSize);
+
+ // test iterator with predicate
+ String firstKey = recordKeys.stream().findFirst().get();
+ recordKeys.remove(firstKey);
+ itr = rocksDBBasedMap.iterator(key -> !key.equals(firstKey));
+ cntSize = 0;
+ while (itr.hasNext()) {
+ HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
+ cntSize++;
+ assert recordKeys.contains(rec.getRecordKey());
+ }
+ assertEquals(recordKeys.size(), cntSize);
}
- assertEquals(recordKeys.size(), cntSize);
}
@Test
public void testSimpleUpsert() throws IOException, URISyntaxException {
Schema schema = HoodieAvroUtils.addMetadataFields(getSimpleSchema());
- RocksDbDiskMap rocksDBBasedMap = new RocksDbDiskMap<>(basePath);
- SchemaTestUtil testUtil = new SchemaTestUtil();
- List<IndexedRecord> insertedRecords = testUtil.generateHoodieTestRecords(0, 100);
- List<String> recordKeys = SpillableMapTestUtils.upsertRecords(insertedRecords, rocksDBBasedMap);
- String oldCommitTime =
- ((GenericRecord) insertedRecords.get(0)).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
-
- // generate updates from inserts for first 50 keys / subset of keys
- List<IndexedRecord> updatedRecords = SchemaTestUtil.updateHoodieTestRecords(recordKeys.subList(0, 50),
- testUtil.generateHoodieTestRecords(0, 50), InProcessTimeGenerator.createNewInstantTime());
- String newCommitTime =
- ((GenericRecord) updatedRecords.get(0)).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
-
- // perform upserts
- List<String> updatedRecordKeys = SpillableMapTestUtils.upsertRecords(updatedRecords, rocksDBBasedMap);
-
- // Upserted records (on disk) should have the latest commit time
- Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = rocksDBBasedMap.iterator();
- while (itr.hasNext()) {
- HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
- try {
- IndexedRecord indexedRecord = (IndexedRecord) rec.getData().getInsertValue(schema).get();
- String latestCommitTime =
- ((GenericRecord) indexedRecord).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
- assert recordKeys.contains(rec.getRecordKey()) || updatedRecordKeys.contains(rec.getRecordKey());
- assertEquals(latestCommitTime, updatedRecordKeys.contains(rec.getRecordKey()) ? newCommitTime : oldCommitTime);
- } catch (IOException io) {
- throw new UncheckedIOException(io);
+ try (RocksDbDiskMap rocksDBBasedMap = new RocksDbDiskMap<>(basePath, new DefaultSerializer<>())) {
+ SchemaTestUtil testUtil = new SchemaTestUtil();
+ List<IndexedRecord> insertedRecords = testUtil.generateHoodieTestRecords(0, 100);
+ List<String> recordKeys = SpillableMapTestUtils.upsertRecords(insertedRecords, rocksDBBasedMap);
+ String oldCommitTime =
+ ((GenericRecord) insertedRecords.get(0)).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
+
+ // generate updates from inserts for first 50 keys / subset of keys
+ List<IndexedRecord> updatedRecords = SchemaTestUtil.updateHoodieTestRecords(recordKeys.subList(0, 50),
+ testUtil.generateHoodieTestRecords(0, 50), InProcessTimeGenerator.createNewInstantTime());
+ String newCommitTime =
+ ((GenericRecord) updatedRecords.get(0)).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
+
+ // perform upserts
+ List<String> updatedRecordKeys = SpillableMapTestUtils.upsertRecords(updatedRecords, rocksDBBasedMap);
+
+ // Upserted records (on disk) should have the latest commit time
+ Iterator<HoodieRecord<? extends HoodieRecordPayload>> itr = rocksDBBasedMap.iterator();
+ while (itr.hasNext()) {
+ HoodieRecord<? extends HoodieRecordPayload> rec = itr.next();
+ try {
+ IndexedRecord indexedRecord = (IndexedRecord) rec.getData().getInsertValue(schema).get();
+ String latestCommitTime =
+ ((GenericRecord) indexedRecord).get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString();
+ assert recordKeys.contains(rec.getRecordKey()) || updatedRecordKeys.contains(rec.getRecordKey());
+ assertEquals(latestCommitTime, updatedRecordKeys.contains(rec.getRecordKey()) ? newCommitTime : oldCommitTime);
+ } catch (IOException io) {
+ throw new UncheckedIOException(io);
+ }
}
}
}
@Test
public void testPutAll() throws IOException, URISyntaxException {
- RocksDbDiskMap<String, HoodieRecord> rocksDBBasedMap = new RocksDbDiskMap<>(basePath);
- SchemaTestUtil testUtil = new SchemaTestUtil();
- List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
- Map<String, HoodieRecord> recordMap = new HashMap<>();
- iRecords.forEach(r -> {
- String key = ((GenericRecord) r).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
- String partitionPath = ((GenericRecord) r).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString();
- HoodieRecord value = new HoodieAvroRecord<>(new HoodieKey(key, partitionPath), new HoodieAvroPayload(Option.of((GenericRecord) r)));
- recordMap.put(key, value);
- });
-
- rocksDBBasedMap.putAll(recordMap);
- // make sure records have spilled to disk
- assertTrue(rocksDBBasedMap.sizeOfFileOnDiskInBytes() > 0);
-
- // make sure all added records are present
- for (Map.Entry<String, HoodieRecord> entry : rocksDBBasedMap.entrySet()) {
- assertTrue(recordMap.containsKey(entry.getKey()));
+ try (RocksDbDiskMap<String, HoodieRecord> rocksDBBasedMap = new RocksDbDiskMap<>(basePath, new DefaultSerializer<>())) {
+ SchemaTestUtil testUtil = new SchemaTestUtil();
+ List<IndexedRecord> iRecords = testUtil.generateHoodieTestRecords(0, 100);
+ Map<String, HoodieRecord> recordMap = new HashMap<>();
+ iRecords.forEach(r -> {
+ String key = ((GenericRecord) r).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
+ String partitionPath = ((GenericRecord) r).get(HoodieRecord.PARTITION_PATH_METADATA_FIELD).toString();
+ HoodieRecord value = new HoodieAvroRecord<>(new HoodieKey(key, partitionPath), new HoodieAvroPayload(Option.of((GenericRecord) r)));
+ recordMap.put(key, value);
+ });
+
+ rocksDBBasedMap.putAll(recordMap);
+ // make sure records have spilled to disk
+ assertTrue(rocksDBBasedMap.sizeOfFileOnDiskInBytes() > 0);
+
+ // make sure all added records are present
+ for (Map.Entry<String, HoodieRecord> entry : rocksDBBasedMap.entrySet()) {
+ assertTrue(recordMap.containsKey(entry.getKey()));
+ }
}
}
@Test
public void testSimpleRemove() throws IOException, URISyntaxException {
- RocksDbDiskMap rocksDBBasedMap = new RocksDbDiskMap<>(basePath);
- List<String> recordKeys = setupMapWithRecords(rocksDBBasedMap, 100);
+ try (RocksDbDiskMap rocksDBBasedMap = new RocksDbDiskMap<>(basePath, new DefaultSerializer<>())) {
+ List<String> recordKeys = setupMapWithRecords(rocksDBBasedMap, 100);
- List<String> deleteKeys = recordKeys.subList(0, 10);
- for (String deleteKey : deleteKeys) {
- assert rocksDBBasedMap.remove(deleteKey) != null;
- assert rocksDBBasedMap.get(deleteKey) == null;
+ List<String> deleteKeys = recordKeys.subList(0, 10);
+ for (String deleteKey : deleteKeys) {
+ assert rocksDBBasedMap.remove(deleteKey) != null;
+ assert rocksDBBasedMap.get(deleteKey) == null;
+ }
}
}
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieMergeOnReadSnapshotReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieMergeOnReadSnapshotReader.java
index d4b57fdc73a..a1b9e76985e 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieMergeOnReadSnapshotReader.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieMergeOnReadSnapshotReader.java
@@ -21,6 +21,7 @@ package org.apache.hudi.hadoop.realtime;
import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner;
import org.apache.hudi.common.util.DefaultSizeEstimator;
import org.apache.hudi.common.util.FileIOUtils;
@@ -121,6 +122,7 @@ public class HoodieMergeOnReadSnapshotReader extends AbstractRealtimeRecordReade
new DefaultSizeEstimator(),
new HoodieRecordSizeEstimator(readerSchema),
jobConf.getEnum(SPILLABLE_DISK_MAP_TYPE.key(), SPILLABLE_DISK_MAP_TYPE.defaultValue()),
+ new DefaultSerializer<>(),
jobConf.getBoolean(DISK_MAP_BITCASK_COMPRESSION_ENABLED.key(), DISK_MAP_BITCASK_COMPRESSION_ENABLED.defaultValue()));
try (ClosableIterator<String> baseFileIterator = baseFileReader.getRecordKeyIterator()) {
timer.startTimer();
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
index 3dff6ad9dee..819ae0c1c52 100644
--- a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
@@ -113,8 +113,8 @@ public class DFSHoodieDatasetInputReader extends DFSDeltaInputReader {
private JavaPairRDD<String, Iterator<FileSlice>> getPartitionToFileSlice(HoodieTableMetaClient metaClient,
List<String> partitionPaths) {
- TableFileSystemView.SliceView fileSystemView = new HoodieTableFileSystemView(metaClient,
- metaClient.getCommitsAndCompactionTimeline().filterCompletedInstants());
+ TableFileSystemView.SliceView fileSystemView = HoodieTableFileSystemView.fileListingBasedFileSystemView(new HoodieSparkEngineContext(jsc),
+ metaClient, metaClient.getCommitsAndCompactionTimeline().filterCompletedInstants());
// pass num partitions to another method
JavaPairRDD<String, Iterator<FileSlice>> partitionToFileSliceList = jsc.parallelize(partitionPaths).mapToPair(p -> {
return new Tuple2<>(p, fileSystemView.getLatestFileSlices(p).iterator());
diff --git a/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/BufferedConnectWriter.java b/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/BufferedConnectWriter.java
index e3ec2a6b502..0c14962db5b 100644
--- a/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/BufferedConnectWriter.java
+++ b/hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/BufferedConnectWriter.java
@@ -23,6 +23,7 @@ import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.serialization.DefaultSerializer;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.util.DefaultSizeEstimator;
import org.apache.hudi.common.util.HoodieRecordSizeEstimator;
@@ -80,6 +81,7 @@ public class BufferedConnectWriter extends AbstractConnectWriter {
new DefaultSizeEstimator(),
new HoodieRecordSizeEstimator(new Schema.Parser().parse(config.getSchema())),
config.getCommonConfig().getSpillableDiskMapType(),
+ new DefaultSerializer<>(),
config.getCommonConfig().isBitCaskDiskMapCompressionEnabled());
} catch (IOException io) {
throw new HoodieIOException("Cannot instantiate an ExternalSpillableMap", io);
diff --git a/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/view/HoodieMetaserverFileSystemView.java b/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/view/HoodieMetaserverFileSystemView.java
index 2ec9dbb9a91..4be9562f258 100644
--- a/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/view/HoodieMetaserverFileSystemView.java
+++ b/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/view/HoodieMetaserverFileSystemView.java
@@ -19,8 +19,10 @@
package org.apache.hudi.common.table.view;
import org.apache.hudi.common.config.HoodieMetaserverConfig;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
import org.apache.hudi.metaserver.client.HoodieMetaserverClient;
import org.apache.hudi.metaserver.client.HoodieMetaserverClientProxy;
@@ -36,7 +38,8 @@ public class HoodieMetaserverFileSystemView extends HoodieTableFileSystemView {
public HoodieMetaserverFileSystemView(HoodieTableMetaClient metaClient,
HoodieTimeline visibleActiveTimeline, HoodieMetaserverConfig config) {
- super(metaClient, visibleActiveTimeline);
+ super(new FileSystemBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getStorageConf()), metaClient.getTableConfig(), metaClient.getStorage(),
+ metaClient.getBasePath().toString()), metaClient, visibleActiveTimeline);
this.metaserverClient = HoodieMetaserverClientProxy.getProxy(config);
this.databaseName = metaClient.getTableConfig().getDatabaseName();
this.tableName = metaClient.getTableConfig().getTableName();
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala
index 1b0ad4557fc..adbabcc7c74 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala
@@ -24,10 +24,8 @@ import org.apache.hudi.HoodieBaseRelation.BaseFileReader
import org.apache.hudi.HoodieConversionUtils.toScalaOption
import org.apache.hudi.HoodieDataSourceHelper.AvroDeserializerSupport
import org.apache.hudi.avro.HoodieAvroUtils
-
import org.apache.hudi.common.config.HoodieCommonConfig.{DISK_MAP_BITCASK_COMPRESSION_ENABLED, SPILLABLE_DISK_MAP_TYPE}
import org.apache.hudi.common.config.{HoodieCommonConfig, HoodieMemoryConfig, HoodieMetadataConfig, TypedProperties}
-
import org.apache.hudi.common.model.{FileSlice, HoodieAvroRecordMerger, HoodieLogFile, HoodieRecord, HoodieRecordMerger, HoodieRecordPayload}
import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.cdc.{HoodieCDCFileSplit, HoodieCDCUtils}
@@ -40,15 +38,13 @@ import org.apache.hudi.common.util.{DefaultSizeEstimator, FileIOUtils}
import org.apache.hudi.common.util.collection.ExternalSpillableMap
import org.apache.hudi.config.HoodiePayloadConfig
import org.apache.hudi.config.HoodieWriteConfig
-
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory
import org.apache.hudi.storage.{StorageConfiguration, StoragePath}
-
import org.apache.avro.Schema
import org.apache.avro.generic.{GenericData, GenericRecord, IndexedRecord}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
-
+import org.apache.hudi.common.serialization.DefaultSerializer
import org.apache.spark.sql.HoodieCatalystExpressionUtils.generateUnsafeProjection
import org.apache.spark.sql.avro.HoodieAvroDeserializer
import org.apache.spark.sql.catalyst.InternalRow
@@ -213,6 +209,7 @@ class CDCFileGroupIterator(split: HoodieCDCFileGroupSplit,
ExternalSpillableMap.DiskMapType.valueOf(props.getString(
SPILLABLE_DISK_MAP_TYPE.key(), SPILLABLE_DISK_MAP_TYPE.defaultValue().toString)
.toUpperCase(Locale.ROOT)),
+ new DefaultSerializer[InternalRow],
props.getBoolean(DISK_MAP_BITCASK_COMPRESSION_ENABLED.key(), DISK_MAP_BITCASK_COMPRESSION_ENABLED.defaultValue()))
private var internalRowToJsonStringConverter = new InternalRowToJsonStringConverter(originTableSchema)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowColumnStatsOverlapProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowColumnStatsOverlapProcedure.scala
index 7a08a686e30..6bd9cacd5de 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowColumnStatsOverlapProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowColumnStatsOverlapProcedure.scala
@@ -258,7 +258,7 @@ class ShowColumnStatsOverlapProcedure extends BaseProcedure with ProcedureBuilde
val basePath = getBasePath(table)
val metaClient = createMetaClient(jsc, basePath)
val storage = metaClient.getStorage
- val globPath = s"$basePath/*/*/*"
+ val globPath = s"$basePath/**"
val statuses = FSUtils.getGlobStatusExcludingMetaFolder(storage, new StoragePath(globPath))
val timeline = metaClient.getActiveTimeline.getCommitsTimeline.filterCompletedInstants()
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableColumnStatsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableColumnStatsProcedure.scala
index 276ccf19110..b09fa654d15 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableColumnStatsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableColumnStatsProcedure.scala
@@ -31,6 +31,7 @@ import org.apache.hudi.metadata.HoodieTableMetadata
import org.apache.hudi.storage.StoragePathInfo
import org.apache.hudi.{AvroConversionUtils, ColumnStatsIndexSupport}
import org.apache.avro.generic.IndexedRecord
+import org.apache.hudi.common.engine.HoodieEngineContext
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -77,10 +78,10 @@ class ShowMetadataTableColumnStatsProcedure extends BaseProcedure with Procedure
val schema = AvroConversionUtils.convertAvroSchemaToStructType(schemaUtil.getTableAvroSchema)
val columnStatsIndex = new ColumnStatsIndexSupport(spark, schema, metadataConfig, metaClient)
val colStatsRecords: HoodieData[HoodieMetadataColumnStats] = columnStatsIndex.loadColumnStatsIndexRecords(targetColumnsSeq, shouldReadInMemory = false)
- val fsView = buildFileSystemView(table)
+ val engineCtx = new HoodieSparkEngineContext(jsc)
+ val fsView = buildFileSystemView(table, engineCtx)
val allFileSlices: Set[FileSlice] = {
if (partitionsSeq.isEmpty) {
- val engineCtx = new HoodieSparkEngineContext(jsc)
val metaTable = HoodieTableMetadata.create(engineCtx, metaClient.getStorage, metadataConfig, basePath)
metaTable.getAllPartitionPaths
.asScala
@@ -130,7 +131,7 @@ class ShowMetadataTableColumnStatsProcedure extends BaseProcedure with Procedure
}
}
- def buildFileSystemView(table: Option[Any]): HoodieTableFileSystemView = {
+ def buildFileSystemView(table: Option[Any], engineContext: HoodieEngineContext): HoodieTableFileSystemView = {
val basePath = getBasePath(table)
val metaClient = createMetaClient(jsc, basePath)
@@ -149,7 +150,7 @@ class ShowMetadataTableColumnStatsProcedure extends BaseProcedure with Procedure
val filteredTimeline = metaClient.getTimelineLayout.getTimelineFactory.createDefaultTimeline(
new java.util.ArrayList[HoodieInstant](instants.toList.asJava).stream(), details)
- new HoodieTableFileSystemView(metaClient, filteredTimeline, new java.util.ArrayList[StoragePathInfo])
+ HoodieTableFileSystemView.fileListingBasedFileSystemView(engineContext, metaClient, filteredTimeline)
}
override def build: Procedure = new ShowMetadataTableColumnStatsProcedure()
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestHoodieMergeHandleWithSparkMerger.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestHoodieMergeHandleWithSparkMerger.java
index c841e7734f8..1f1fbae8b11 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestHoodieMergeHandleWithSparkMerger.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestHoodieMergeHandleWithSparkMerger.java
@@ -197,7 +197,7 @@ public class TestHoodieMergeHandleWithSparkMerger extends SparkClientFunctionalT
}
public HoodieTableFileSystemView getFileSystemView() {
- return new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline());
+ return HoodieTableFileSystemView.fileListingBasedFileSystemView(context(), metaClient, metaClient.getActiveTimeline());
}
public List<FileSlice> getLatestFileSlices(String partitionPath) {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
index 73e104db8d4..02a6a95360e 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
@@ -95,7 +95,6 @@ import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
import org.apache.hudi.metadata.HoodieBackedTableMetadata;
import org.apache.hudi.metadata.HoodieBackedTableMetadataWriter;
-import org.apache.hudi.metadata.HoodieMetadataFileSystemView;
import org.apache.hudi.metadata.HoodieMetadataLogRecordReader;
import org.apache.hudi.metadata.HoodieMetadataMetrics;
import org.apache.hudi.metadata.HoodieMetadataPayload;
@@ -796,7 +795,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
final Map<String, MetadataPartitionType> metadataEnabledPartitionTypes = new HashMap<>();
metadataWriter.getEnabledPartitionTypes().forEach(e -> metadataEnabledPartitionTypes.put(e.getPartitionPath(), e));
- HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metadataMetaClient, metadataMetaClient.getActiveTimeline());
+ HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(engineContext, metadataMetaClient, metadataMetaClient.getActiveTimeline());
metadataTablePartitions.forEach(partition -> {
List<FileSlice> latestSlices = fsView.getLatestFileSlices(partition).collect(Collectors.toList());
if (COLUMN_STATS.getPartitionPath().equals(partition)) {
@@ -3375,9 +3374,9 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTableMetadata tableMetadata = metadata(client);
- HoodieMetadataFileSystemView metadataFileSystemView = new HoodieMetadataFileSystemView(
- metaClient, metaClient.reloadActiveTimeline(), tableMetadata);
- HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline());
+ HoodieTableFileSystemView metadataFileSystemView = new HoodieTableFileSystemView(tableMetadata,
+ metaClient, metaClient.reloadActiveTimeline());
+ HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(getEngineContext(), metaClient, metaClient.getActiveTimeline());
tableMetadata.getAllPartitionPaths().forEach(partition -> {
List<String> fileNamesFromMetadataFileListing = metadataFileSystemView.getLatestBaseFiles(partition)
.map(baseFile -> baseFile.getFileName())
@@ -3721,7 +3720,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// Metadata table should automatically compact and clean
// versions are +1 as autoclean / compaction happens end of commits
int numFileVersions = metadataWriteConfig.getCleanerFileVersionsRetained() + 1;
- HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metadataMetaClient, metadataMetaClient.getActiveTimeline());
+ HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(engineContext, metadataMetaClient, metadataMetaClient.getActiveTimeline());
metadataTablePartitions.forEach(partition -> {
List<FileSlice> latestSlices = fsView.getLatestFileSlices(partition).collect(Collectors.toList());
assertTrue(latestSlices.stream().map(FileSlice::getBaseFile).count() <= latestSlices.size(), "Should have a single latest base file per file group");
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestMetadataUtilRLIandSIRecordGeneration.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestMetadataUtilRLIandSIRecordGeneration.java
index 85aa85218c0..6bc29a16388 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestMetadataUtilRLIandSIRecordGeneration.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestMetadataUtilRLIandSIRecordGeneration.java
@@ -37,6 +37,7 @@ import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.TableSchemaResolver;
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.RawTripTestPayload;
import org.apache.hudi.common.util.Option;
@@ -46,8 +47,8 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.metadata.BaseFileRecordParsingUtils;
-import org.apache.hudi.metadata.HoodieMetadataFileSystemView;
import org.apache.hudi.metadata.HoodieMetadataPayload;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.testutils.HoodieClientTestBase;
@@ -326,10 +327,11 @@ public class TestMetadataUtilRLIandSIRecordGeneration extends HoodieClientTestBa
.withIndexOptions(Collections.emptyMap())
.build();
HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).withSecondaryIndexParallelism(2).build();
- HoodieMetadataFileSystemView metadataView = new HoodieMetadataFileSystemView(engineContext, metaClient, metaClient.getActiveTimeline(), metadataConfig);
+ HoodieTableMetadata metadata = HoodieTableMetadata.create(engineContext, storage, metadataConfig, metaClient.getBasePath().toString());
+ HoodieTableFileSystemView metadataView = new HoodieTableFileSystemView(metadata, metaClient, metaClient.getActiveTimeline());
metadataView.loadAllPartitions();
List<Pair<String, FileSlice>> partitionFileSlicePairs = new ArrayList<>();
- HoodieMetadataFileSystemView finalMetadataView = metadataView;
+ HoodieTableFileSystemView finalMetadataView = metadataView;
Arrays.asList(dataGen.getPartitionPaths()).forEach(partition -> finalMetadataView.getLatestMergedFileSlicesBeforeOrOn(partition, firstCommitTime)
.forEach(fs -> partitionFileSlicePairs.add(Pair.of(partition, fs))));
List<HoodieRecord> secondaryIndexRecords = readSecondaryKeysFromFileSlices(
@@ -352,7 +354,8 @@ public class TestMetadataUtilRLIandSIRecordGeneration extends HoodieClientTestBa
// assert SI
String secondCommitTime = commitTime;
metaClient = HoodieTableMetaClient.reload(metaClient);
- metadataView = new HoodieMetadataFileSystemView(engineContext, metaClient, metaClient.getActiveTimeline(), metadataConfig);
+ metadata.reset();
+ metadataView = new HoodieTableFileSystemView(metadata, metaClient, metaClient.getActiveTimeline());
List<HoodieWriteStat> allWriteStats = writeStatuses2.collect().stream().map(WriteStatus::getStat).collect(Collectors.toList());
secondaryIndexRecords =
convertWriteStatsToSecondaryIndexRecords(allWriteStats, secondCommitTime, indexDefinition, metadataConfig, metadataView, metaClient, engineContext, EngineType.SPARK).collectAsList();
@@ -386,7 +389,8 @@ public class TestMetadataUtilRLIandSIRecordGeneration extends HoodieClientTestBa
// assert SI
String thirdCommitTime = commitTime;
metaClient = HoodieTableMetaClient.reload(metaClient);
- metadataView = new HoodieMetadataFileSystemView(engineContext, metaClient, metaClient.getActiveTimeline(), metadataConfig);
+ metadata.reset();
+ metadataView = new HoodieTableFileSystemView(metadata, metaClient, metaClient.getActiveTimeline());
allWriteStats = writeStatuses3.collect().stream().map(WriteStatus::getStat).collect(Collectors.toList());
secondaryIndexRecords =
convertWriteStatsToSecondaryIndexRecords(allWriteStats, thirdCommitTime, indexDefinition, metadataConfig, metadataView, metaClient, engineContext, EngineType.SPARK).collectAsList();
@@ -417,7 +421,8 @@ public class TestMetadataUtilRLIandSIRecordGeneration extends HoodieClientTestBa
// assert SI
String fourthCommitTime = commitTime;
metaClient = HoodieTableMetaClient.reload(metaClient);
- metadataView = new HoodieMetadataFileSystemView(engineContext, metaClient, metaClient.getActiveTimeline(), metadataConfig);
+ metadata.reset();
+ metadataView = new HoodieTableFileSystemView(metadata, metaClient, metaClient.getActiveTimeline());
allWriteStats = writeStatuses4.collect().stream().map(WriteStatus::getStat).collect(Collectors.toList());
secondaryIndexRecords =
convertWriteStatsToSecondaryIndexRecords(allWriteStats, fourthCommitTime, indexDefinition, metadataConfig, metadataView, metaClient, engineContext, EngineType.SPARK).collectAsList();
@@ -440,7 +445,8 @@ public class TestMetadataUtilRLIandSIRecordGeneration extends HoodieClientTestBa
// assert SI
String fifthCommitTime = commitTime;
metaClient = HoodieTableMetaClient.reload(metaClient);
- metadataView = new HoodieMetadataFileSystemView(engineContext, metaClient, metaClient.getActiveTimeline(), metadataConfig);
+ metadata.reset();
+ metadataView = new HoodieTableFileSystemView(metadata, metaClient, metaClient.getActiveTimeline());
allWriteStats = writeStatuses5.collect().stream().map(WriteStatus::getStat).collect(Collectors.toList());
secondaryIndexRecords =
convertWriteStatsToSecondaryIndexRecords(allWriteStats, fifthCommitTime, indexDefinition, metadataConfig, metadataView, metaClient, engineContext, EngineType.SPARK).collectAsList();
@@ -456,7 +462,8 @@ public class TestMetadataUtilRLIandSIRecordGeneration extends HoodieClientTestBa
HoodieCommitMetadata compactionCommitMetadata = (HoodieCommitMetadata) compactionWriteMetadata.getCommitMetadata().get();
// assert SI records
metaClient = HoodieTableMetaClient.reload(metaClient);
- metadataView = new HoodieMetadataFileSystemView(engineContext, metaClient, metaClient.getActiveTimeline(), metadataConfig);
+ metadata.reset();
+ metadataView = new HoodieTableFileSystemView(metadata, metaClient, metaClient.getActiveTimeline());
allWriteStats = compactionCommitMetadata.getWriteStats();
secondaryIndexRecords = convertWriteStatsToSecondaryIndexRecords(
allWriteStats, compactionInstantOpt.get(), indexDefinition, metadataConfig, metadataView, metaClient, engineContext, EngineType.SPARK).collectAsList();
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
index c9bca4cd163..9b8e38611dc 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
@@ -43,7 +43,6 @@ import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.metadata.HoodieBackedTestDelayedTableMetadata;
-import org.apache.hudi.metadata.HoodieMetadataFileSystemView;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
import org.apache.hudi.timeline.service.TimelineService;
@@ -78,7 +77,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Tests the {@link RemoteHoodieTableFileSystemView} with metadata table enabled, using
- * {@link HoodieMetadataFileSystemView} on the timeline server.
+ * {@link org.apache.hudi.common.table.view.HoodieTableFileSystemView} on the timeline server.
*/
public class TestRemoteFileSystemViewWithMetadataTable extends HoodieSparkClientTestHarness {
private static final Logger LOG = LoggerFactory.getLogger(TestRemoteFileSystemViewWithMetadataTable.class);
@@ -118,7 +117,7 @@ public class TestRemoteFileSystemViewWithMetadataTable extends HoodieSparkClient
.serverPort(config.getViewStorageConfig().getRemoteViewServerPort()).build(),
HoodieStorageUtils.getStorage(getDefaultStorageConf()),
FileSystemViewManager.createViewManager(
- context, config.getViewStorageConfig(),
+ context, config.getMetadataConfig(), config.getViewStorageConfig(),
config.getCommonConfig(),
metaClient -> new HoodieBackedTestDelayedTableMetadata(
context, metaClient.getStorage(), config.getMetadataConfig(), metaClient.getBasePath().toString(), true)));
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala
index 7c4a836f200..0bbd11021c4 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieFileIndex.scala
@@ -41,6 +41,7 @@ import org.apache.hudi.keygen.TimestampBasedAvroKeyGenerator.TimestampType
import org.apache.hudi.keygen.constant.KeyGeneratorType
import org.apache.hudi.metadata.HoodieTableMetadata
import org.apache.hudi.storage.StoragePath
+import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration
import org.apache.hudi.testutils.HoodieSparkClientTestBase
import org.apache.hudi.util.JFunction
import org.apache.spark.sql._
@@ -831,7 +832,8 @@ class TestHoodieFileIndex extends HoodieSparkClientTestBase with ScalaAssertionS
private def getFileCountInPartitionPath(partitionPath: String): Int = {
metaClient.reloadActiveTimeline()
val activeInstants = metaClient.getActiveTimeline.getCommitsTimeline.filterCompletedInstants
- val fileSystemView = new HoodieTableFileSystemView(metaClient, activeInstants)
+ val fileSystemView = HoodieTableFileSystemView.fileListingBasedFileSystemView(
+ new HoodieJavaEngineContext(new HadoopStorageConfiguration(false)), metaClient, activeInstants)
fileSystemView.getAllBaseFiles(partitionPath).iterator().asScala.toSeq.length
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBloomFiltersIndexSupport.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBloomFiltersIndexSupport.scala
index cb0419cd8d1..cdffb0c6564 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBloomFiltersIndexSupport.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBloomFiltersIndexSupport.scala
@@ -22,14 +22,13 @@ package org.apache.hudi.functional
import org.apache.hudi.DataSourceWriteOptions._
import org.apache.hudi.common.config.{HoodieMetadataConfig, TypedProperties}
import org.apache.hudi.common.model.{FileSlice, HoodieTableType}
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings
import org.apache.hudi.config.HoodieWriteConfig
-import org.apache.hudi.metadata.HoodieMetadataFileSystemView
import org.apache.hudi.testutils.HoodieSparkClientTestBase
import org.apache.hudi.util.{JFunction, JavaConversions}
import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieFileIndex}
-
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, Literal}
import org.apache.spark.sql.functions.{col, not}
import org.apache.spark.sql.types.StringType
@@ -41,7 +40,6 @@ import org.junit.jupiter.params.provider.EnumSource
import java.util.concurrent.atomic.AtomicInteger
import java.util.stream.Collectors
-
import scala.collection.JavaConverters._
import scala.collection.{JavaConverters, mutable}
@@ -163,8 +161,8 @@ class TestBloomFiltersIndexSupport extends HoodieSparkClientTestBase {
totalLatestDataFiles
}
- private def getTableFileSystemView(opts: Map[String, String]): HoodieMetadataFileSystemView = {
- new HoodieMetadataFileSystemView(metaClient, metaClient.getActiveTimeline, metadataWriter(getWriteConfig(opts)).getTableMetadata)
+ private def getTableFileSystemView(opts: Map[String, String]): HoodieTableFileSystemView = {
+ new HoodieTableFileSystemView(metadataWriter(getWriteConfig(opts)).getTableMetadata, metaClient, metaClient.getActiveTimeline)
}
private def doWriteAndValidateBloomFilters(hudiOpts: Map[String, String],
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndexWithSQL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndexWithSQL.scala
index ce1771886e1..66478c16cd3 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndexWithSQL.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndexWithSQL.scala
@@ -27,18 +27,18 @@ import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.model.{HoodieCommitMetadata, HoodieTableType, WriteOperationType}
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
import org.apache.hudi.common.table.timeline.{HoodieInstant, MetadataConversionUtils}
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView
import org.apache.hudi.common.util.FileIOUtils
import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig}
import org.apache.hudi.functional.ColumnStatIndexTestBase.{ColumnStatsTestCase, ColumnStatsTestParams}
import org.apache.hudi.index.HoodieIndex.IndexType.INMEMORY
-import org.apache.hudi.metadata.HoodieMetadataFileSystemView
+import org.apache.hudi.metadata.HoodieBackedTableMetadata
import org.apache.hudi.util.JavaConversions
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, BitwiseOr, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, LessThanOrEqual, Literal, Or}
-import org.apache.spark.sql.hudi.ColumnStatsExpressionUtils.{AllowedTransformationExpression, swapAttributeRefInExpr}
-import org.apache.spark.sql.hudi.{ColumnStatsExpressionUtils, DataSkippingUtils}
+import org.apache.spark.sql.hudi.DataSkippingUtils
import org.apache.spark.sql.types.StringType
import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
import org.junit.jupiter.api.Test
@@ -46,7 +46,6 @@ import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import java.io.File
-import java.util.concurrent.atomic.AtomicBoolean
import scala.collection.JavaConverters._
class TestColumnStatsIndexWithSQL extends ColumnStatIndexTestBase {
@@ -669,8 +668,11 @@ class TestColumnStatsIndexWithSQL extends ColumnStatIndexTestBase {
totalLatestDataFiles
}
- private def getTableFileSystemView(opts: Map[String, String]): HoodieMetadataFileSystemView = {
- new HoodieMetadataFileSystemView(metaClient, metaClient.getActiveTimeline, metadataWriter(getWriteConfig(opts)).getTableMetadata)
+ private def getTableFileSystemView(opts: Map[String, String]): HoodieTableFileSystemView = {
+ val writeConfig = getWriteConfig(opts)
+ val metadataTable = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), metaClient.getStorage,
+ writeConfig.getMetadataConfig, writeConfig.getBasePath)
+ new HoodieTableFileSystemView(metadataTable, metaClient, metaClient.getActiveTimeline)
}
protected def getWriteConfig(hudiOpts: Map[String, String]): HoodieWriteConfig = {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieMultipleBaseFileFormat.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieMultipleBaseFileFormat.scala
index c336a21f54a..8cce89ba7d9 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieMultipleBaseFileFormat.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieMultipleBaseFileFormat.scala
@@ -111,7 +111,7 @@ class TestHoodieMultipleBaseFileFormat extends HoodieSparkClientTestBase with Sp
val engineContext: HoodieEngineContext = new HoodieLocalEngineContext(storageConf)
val metadataConfig: HoodieMetadataConfig = HoodieMetadataConfig.newBuilder.build
val viewManager: FileSystemViewManager = FileSystemViewManager.createViewManager(
- engineContext, FileSystemViewStorageConfig.newBuilder.build,
+ engineContext, metadataConfig, FileSystemViewStorageConfig.newBuilder.build,
HoodieCommonConfig.newBuilder.build,
(mc: HoodieTableMetaClient) => HoodieTableMetadata.create(engineContext, mc.getStorage, metadataConfig, basePath))
val fsView: SyncableFileSystemView = viewManager.getFileSystemView(metaClient)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala
index 0e31499e05c..d88ff3dd9d3 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala
@@ -28,11 +28,12 @@ import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.model.{FileSlice, HoodieBaseFile, HoodieCommitMetadata, HoodieFailedWritesCleaningPolicy, HoodieTableType, WriteConcurrencyMode, WriteOperationType}
import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.HoodieInstant
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView
import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings
import org.apache.hudi.config.{HoodieCleanConfig, HoodieClusteringConfig, HoodieCompactionConfig, HoodieLockConfig, HoodieWriteConfig}
import org.apache.hudi.exception.{HoodieException, HoodieWriteConflictException}
import org.apache.hudi.keygen.constant.KeyGeneratorOptions
-import org.apache.hudi.metadata.{HoodieBackedTableMetadata, HoodieMetadataFileSystemView, MetadataPartitionType}
+import org.apache.hudi.metadata.{HoodieBackedTableMetadata, MetadataPartitionType}
import org.apache.hudi.util.{JFunction, JavaConversions}
import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieFileIndex, PartitionStatsIndexSupport}
import org.apache.spark.sql.SaveMode
@@ -665,8 +666,8 @@ class TestPartitionStatsIndex extends PartitionStatsIndexTestBase {
totalLatestDataFiles
}
- private def getTableFileSystemView(opts: Map[String, String]): HoodieMetadataFileSystemView = {
- new HoodieMetadataFileSystemView(metaClient, metaClient.getActiveTimeline, metadataWriter(getWriteConfig(opts)).getTableMetadata)
+ private def getTableFileSystemView(opts: Map[String, String]): HoodieTableFileSystemView = {
+ new HoodieTableFileSystemView(metadataWriter(getWriteConfig(opts)).getTableMetadata, metaClient, metaClient.getActiveTimeline)
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala
index 16b076fc8f4..5973c52cf55 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala
@@ -22,14 +22,14 @@ package org.apache.hudi.functional
import org.apache.hudi.client.common.HoodieSparkEngineContext
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.model.{FileSlice, WriteOperationType}
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
import org.apache.hudi.common.testutils.HoodieTestUtils
import org.apache.hudi.metadata.HoodieTableMetadataUtil.getPartitionStatsIndexKey
import org.apache.hudi.metadata.MetadataPartitionType.PARTITION_STATS
-import org.apache.hudi.metadata.{HoodieMetadataFileSystemView, MetadataPartitionType}
+import org.apache.hudi.metadata.{HoodieBackedTableMetadata, MetadataPartitionType}
import org.apache.hudi.util.JFunction
import org.apache.hudi.{DataSourceReadOptions, HoodieFileIndex}
-import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, Expression, GreaterThan, IsNotNull, LessThan, Literal, Or}
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
import org.apache.spark.sql.types.{IntegerType, StringType}
@@ -669,7 +669,7 @@ class TestPartitionStatsIndexWithSql extends HoodieSparkSqlTestBase {
private def getLatestDataFilesCount(includeLogFiles: Boolean = true, metaClient: HoodieTableMetaClient) = {
var totalLatestDataFiles = 0L
- val fsView: HoodieMetadataFileSystemView = getTableFileSystemView(metaClient)
+ val fsView: HoodieTableFileSystemView = getTableFileSystemView(metaClient)
try {
fsView.getAllLatestFileSlicesBeforeOrOn(metaClient.getActiveTimeline.lastInstant().get().requestedTime)
.values()
@@ -683,12 +683,14 @@ class TestPartitionStatsIndexWithSql extends HoodieSparkSqlTestBase {
totalLatestDataFiles
}
- private def getTableFileSystemView(metaClient: HoodieTableMetaClient): HoodieMetadataFileSystemView = {
- new HoodieMetadataFileSystemView(
- new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext)),
+ private def getTableFileSystemView(metaClient: HoodieTableMetaClient): HoodieTableFileSystemView = {
+ val metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexPartitionStats(true).build()
+ val metadataTable = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(spark.sparkContext),
+ metaClient.getStorage, metadataConfig, metaClient.getBasePath.toString)
+ new HoodieTableFileSystemView(
+ metadataTable,
metaClient,
- metaClient.getActiveTimeline,
- HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexPartitionStats(true).build())
+ metaClient.getActiveTimeline)
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
index d7fe8894cbd..a7e95e8b41f 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
@@ -28,6 +28,7 @@ import org.apache.hudi.common.config.{HoodieMetadataConfig, RecordMergeMode, Typ
import org.apache.hudi.common.model._
import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.HoodieInstant
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView
import org.apache.hudi.common.testutils.HoodieTestUtils
import org.apache.hudi.config._
import org.apache.hudi.exception.{HoodieMetadataIndexException, HoodieWriteConflictException}
@@ -1754,7 +1755,7 @@ class TestSecondaryIndexPruning extends SparkClientFunctionalTestHarness {
private def getLatestDataFilesCount(opts: Map[String, String], includeLogFiles: Boolean = true) = {
var totalLatestDataFiles = 0L
- val fsView: HoodieMetadataFileSystemView = getTableFileSystemView(opts)
+ val fsView: HoodieTableFileSystemView = getTableFileSystemView(opts)
try {
fsView.getAllLatestFileSlicesBeforeOrOn(metaClient.getActiveTimeline.lastInstant().get().requestedTime)
.values()
@@ -1768,9 +1769,9 @@ class TestSecondaryIndexPruning extends SparkClientFunctionalTestHarness {
totalLatestDataFiles
}
- private def getTableFileSystemView(opts: Map[String, String]): HoodieMetadataFileSystemView = {
- new HoodieMetadataFileSystemView(metaClient, metaClient.getActiveTimeline,
- new HoodieBackedTableMetadata(context(), metaClient.getStorage, getWriteConfig(opts).getMetadataConfig, metaClient.getBasePath.toString, true))
+ private def getTableFileSystemView(opts: Map[String, String]): HoodieTableFileSystemView = {
+ val metadataTable = new HoodieBackedTableMetadata(context(), metaClient.getStorage, getWriteConfig(opts).getMetadataConfig, metaClient.getBasePath.toString, true);
+ new HoodieTableFileSystemView(metadataTable, metaClient, metaClient.getActiveTimeline)
}
private def getWriteConfig(hudiOpts: Map[String, String]): HoodieWriteConfig = {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSixToFiveDowngradeHandler.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSixToFiveDowngradeHandler.scala
index f8a45c7b679..b1cc3fe882f 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSixToFiveDowngradeHandler.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSixToFiveDowngradeHandler.scala
@@ -25,7 +25,6 @@ import org.apache.hudi.common.model.HoodieTableType
import org.apache.hudi.common.table.view.HoodieTableFileSystemView
import org.apache.hudi.common.table.{HoodieTableMetaClient, HoodieTableVersion}
import org.apache.hudi.config.HoodieCompactionConfig
-import org.apache.hudi.metadata.HoodieMetadataFileSystemView
import org.apache.hudi.storage.StoragePath
import org.apache.hudi.table.upgrade.{SparkUpgradeDowngradeHelper, UpgradeDowngrade}
import org.apache.spark.sql.SaveMode
@@ -131,9 +130,9 @@ class TestSixToFiveDowngradeHandler extends RecordLevelIndexTestBase {
private def getTableFileSystemView(opts: Map[String, String]): HoodieTableFileSystemView = {
if (metaClient.getTableConfig.isMetadataTableAvailable) {
- new HoodieMetadataFileSystemView(metaClient, metaClient.getActiveTimeline, metadataWriter(getWriteConfig(opts)).getTableMetadata)
+ new HoodieTableFileSystemView(metadataWriter(getWriteConfig(opts)).getTableMetadata, metaClient, metaClient.getActiveTimeline)
} else {
- new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline)
+ HoodieTableFileSystemView.fileListingBasedFileSystemView(context, metaClient, metaClient.getActiveTimeline)
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestExpressionIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestExpressionIndex.scala
index d0f1a6307cb..c478439308d 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestExpressionIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestExpressionIndex.scala
@@ -28,7 +28,7 @@ import org.apache.hudi.common.config.{HoodieMetadataConfig, HoodieStorageConfig,
import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.model.FileSlice
import org.apache.hudi.common.table.HoodieTableMetaClient
-import org.apache.hudi.common.table.view.FileSystemViewManager
+import org.apache.hudi.common.table.view.{FileSystemViewManager, HoodieTableFileSystemView}
import org.apache.hudi.common.testutils.HoodieTestUtils
import org.apache.hudi.common.util.Option
import org.apache.hudi.config.{HoodieCleanConfig, HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig}
@@ -37,7 +37,7 @@ import org.apache.hudi.hive.{HiveSyncTool, HoodieHiveSyncClient}
import org.apache.hudi.index.HoodieIndex
import org.apache.hudi.index.expression.HoodieExpressionIndex
import org.apache.hudi.metadata.HoodieTableMetadataUtil.getPartitionStatsIndexKey
-import org.apache.hudi.metadata.{HoodieMetadataFileSystemView, MetadataPartitionType}
+import org.apache.hudi.metadata.{HoodieBackedTableMetadata, MetadataPartitionType}
import org.apache.hudi.storage.StoragePath
import org.apache.hudi.sync.common.HoodieSyncConfig.{META_SYNC_BASE_PATH, META_SYNC_DATABASE_NAME, META_SYNC_NO_PARTITION_METADATA, META_SYNC_TABLE_NAME}
import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
@@ -2243,7 +2243,7 @@ class TestExpressionIndex extends HoodieSparkSqlTestBase {
private def getLatestDataFilesCount(includeLogFiles: Boolean = true, metaClient: HoodieTableMetaClient) = {
var totalLatestDataFiles = 0L
- val fsView: HoodieMetadataFileSystemView = getTableFileSystemView(metaClient)
+ val fsView: HoodieTableFileSystemView = getTableFileSystemView(metaClient)
try {
fsView.getAllLatestFileSlicesBeforeOrOn(metaClient.getActiveTimeline.lastInstant().get().requestedTime)
.values()
@@ -2257,12 +2257,14 @@ class TestExpressionIndex extends HoodieSparkSqlTestBase {
totalLatestDataFiles
}
- private def getTableFileSystemView(metaClient: HoodieTableMetaClient): HoodieMetadataFileSystemView = {
- new HoodieMetadataFileSystemView(
- new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext)),
+ private def getTableFileSystemView(metaClient: HoodieTableMetaClient): HoodieTableFileSystemView = {
+ val engineContext = new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext))
+ val metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexPartitionStats(true).build()
+ val metadataTable = new HoodieBackedTableMetadata(engineContext, metaClient.getStorage, metadataConfig, metaClient.getBasePath.toString)
+ new HoodieTableFileSystemView(
+ metadataTable,
metaClient,
- metaClient.getActiveTimeline,
- HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexPartitionStats(true).build())
+ metaClient.getActiveTimeline)
}
private def getWriteConfig(hudiOpts: Map[String, String], basePath: String): HoodieWriteConfig = {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
index df81df34cf9..f321d09039d 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
@@ -534,7 +534,7 @@ class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
val metadataConfig = HoodieMetadataConfig.newBuilder.build
val engineContext = new HoodieLocalEngineContext(storageConf)
val viewManager: FileSystemViewManager = FileSystemViewManager.createViewManager(
- engineContext, FileSystemViewStorageConfig.newBuilder.build,
+ engineContext, metadataConfig, FileSystemViewStorageConfig.newBuilder.build,
HoodieCommonConfig.newBuilder.build,
(v1: HoodieTableMetaClient) => {
HoodieTableMetadata.create(
diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
index 481fa02d1e6..7fbed5e466c 100644
--- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
+++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
@@ -409,20 +409,20 @@ public class TimelineService {
case MEMORY:
FileSystemViewStorageConfig.Builder inMemConfBuilder = FileSystemViewStorageConfig.newBuilder();
inMemConfBuilder.withStorageType(FileSystemViewStorageType.MEMORY);
- return FileSystemViewManager.createViewManager(localEngineContext, inMemConfBuilder.build(), commonConfig);
+ return FileSystemViewManager.createViewManager(localEngineContext, metadataConfig, inMemConfBuilder.build(), commonConfig);
case SPILLABLE_DISK: {
FileSystemViewStorageConfig.Builder spillableConfBuilder = FileSystemViewStorageConfig.newBuilder();
spillableConfBuilder.withStorageType(FileSystemViewStorageType.SPILLABLE_DISK)
.withBaseStoreDir(config.baseStorePathForFileGroups)
.withMaxMemoryForView(config.maxViewMemPerTableInMB * 1024 * 1024L)
.withMemFractionForPendingCompaction(config.memFractionForCompactionPerTable);
- return FileSystemViewManager.createViewManager(localEngineContext, spillableConfBuilder.build(), commonConfig);
+ return FileSystemViewManager.createViewManager(localEngineContext, metadataConfig, spillableConfBuilder.build(), commonConfig);
}
case EMBEDDED_KV_STORE: {
FileSystemViewStorageConfig.Builder rocksDBConfBuilder = FileSystemViewStorageConfig.newBuilder();
rocksDBConfBuilder.withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE)
.withRocksDBPath(config.rocksDBPath);
- return FileSystemViewManager.createViewManager(localEngineContext, rocksDBConfBuilder.build(), commonConfig);
+ return FileSystemViewManager.createViewManager(localEngineContext, metadataConfig, rocksDBConfBuilder.build(), commonConfig);
}
default:
throw new IllegalArgumentException("Invalid view manager storage type :" + config.viewStorageType);
diff --git a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java
index 1f532d0827c..9b83901c3f6 100644
--- a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java
+++ b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java
@@ -19,6 +19,7 @@
package org.apache.hudi.timeline.service.functional;
import org.apache.hudi.common.config.HoodieCommonConfig;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.model.HoodieFileGroup;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
@@ -74,7 +75,7 @@ public class TestRemoteHoodieTableFileSystemView extends TestHoodieTableFileSyst
server = new TimelineService(localEngineContext, HadoopFSUtils.getStorageConf(),
TimelineService.Config.builder().serverPort(0).build(),
HoodieStorageUtils.getStorage(getDefaultStorageConf()),
- FileSystemViewManager.createViewManager(localEngineContext, sConf, commonConfig));
+ FileSystemViewManager.createViewManager(localEngineContext, HoodieMetadataConfig.newBuilder().build(), sConf, commonConfig));
server.startService();
} catch (Exception ex) {
throw new RuntimeException(ex);
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
index 8b0ce2a6ee7..674fa356c35 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
@@ -60,7 +60,6 @@ import org.apache.hudi.common.table.view.FileSystemViewManager;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.table.view.FileSystemViewStorageType;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
-import org.apache.hudi.common.table.view.SpillableMapBasedFileSystemView;
import org.apache.hudi.common.util.CleanerUtils;
import org.apache.hudi.common.util.ConfigUtils;
import org.apache.hudi.common.util.FileFormatUtils;
@@ -838,11 +837,11 @@ public class HoodieMetadataTableValidator implements Serializable {
/**
* Compare the file listing and index data between metadata table and fileSystem.
* For now, validate five kinds of apis:
- * 1. HoodieMetadataFileSystemView::getLatestFileSlices
- * 2. HoodieMetadataFileSystemView::getLatestBaseFiles
- * 3. HoodieMetadataFileSystemView::getAllFileGroups and HoodieMetadataFileSystemView::getAllFileSlices
- * 4. HoodieBackedTableMetadata::getColumnStats
- * 5. HoodieBackedTableMetadata::getBloomFilters
+ * 1. HoodieTableFileSystemView::getLatestFileSlices
+ * 2. HoodieTableFileSystemView::getLatestBaseFiles
+ * 3. HoodieTableFileSystemView::getAllFileGroups and HoodieTableFileSystemView::getAllFileSlices
+ * 4. HoodieTableFileSystemView::getColumnStats
+ * 5. HoodieTableFileSystemView::getBloomFilters
*
* @param metadataTableBasedContext Validation context containing information based on metadata table
* @param fsBasedContext Validation context containing information based on the file system
@@ -1760,10 +1759,10 @@ public class HoodieMetadataTableValidator implements Serializable {
FileSystemViewStorageConfig viewConf = FileSystemViewStorageConfig.newBuilder().fromProperties(props).build();
ValidationUtils.checkArgument(viewConf.getStorageType().name().equals(viewStorageType), "View storage type not reflected");
HoodieCommonConfig commonConfig = HoodieCommonConfig.newBuilder().fromProperties(props).build();
- this.fileSystemView = getFileSystemView(engineContext,
- metaClient, metadataConfig, viewConf, commonConfig);
this.tableMetadata = HoodieTableMetadata.create(
engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePath().toString());
+ this.fileSystemView = getFileSystemView(engineContext,
+ metaClient, metadataConfig, viewConf, commonConfig);
if (metaClient.getCommitsTimeline().filterCompletedInstants().countInstants() > 0) {
this.allColumnNameList = getAllColumnNames();
}
@@ -1775,18 +1774,17 @@ public class HoodieMetadataTableValidator implements Serializable {
private HoodieTableFileSystemView getFileSystemView(HoodieEngineContext context,
HoodieTableMetaClient metaClient, HoodieMetadataConfig metadataConfig,
FileSystemViewStorageConfig viewConf, HoodieCommonConfig commonConfig) {
- HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
switch (viewConf.getStorageType()) {
case SPILLABLE_DISK:
LOG.debug("Creating Spillable Disk based Table View");
- return new SpillableMapBasedFileSystemView(metaClient, timeline, viewConf, commonConfig);
+ break;
case MEMORY:
LOG.debug("Creating in-memory based Table View");
- return FileSystemViewManager.createInMemoryFileSystemView(context,
- metaClient, metadataConfig);
+ break;
default:
throw new HoodieException("Unsupported storage type " + viewConf.getStorageType() + ", used with HoodieMetadataTableValidator");
}
+ return (HoodieTableFileSystemView) FileSystemViewManager.createViewManager(context, metadataConfig, viewConf, commonConfig, unused -> tableMetadata).getFileSystemView(metaClient);
}
public HoodieTableMetaClient getMetaClient() {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
index d625e7dafe8..115e229cb44 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
@@ -90,9 +90,9 @@ public class HoodieSnapshotCopier implements Serializable {
final StorageConfiguration<?> storageConf = HadoopFSUtils.getStorageConfWithCopy(jsc.hadoopConfiguration());
final HoodieTableMetaClient tableMetadata = HoodieTableMetaClient.builder()
.setConf(HadoopFSUtils.getStorageConfWithCopy(fs.getConf())).setBasePath(baseDir).build();
- final BaseFileOnlyView fsView = new HoodieTableFileSystemView(tableMetadata,
- tableMetadata.getActiveTimeline().getWriteTimeline().filterCompletedInstants());
HoodieEngineContext context = new HoodieSparkEngineContext(jsc);
+ final BaseFileOnlyView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context, tableMetadata,
+ tableMetadata.getActiveTimeline().getWriteTimeline().filterCompletedInstants());
// Get the latest commit
Option<HoodieInstant> latestCommit =
tableMetadata.getActiveTimeline().getWriteTimeline().filterCompletedInstants().lastInstant();
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
index 0bedc070718..0a09e678341 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
@@ -210,7 +210,7 @@ public class HoodieSnapshotExporter {
HoodieEngineContext context = new HoodieSparkEngineContext(jsc);
context.setJobStatus(this.getClass().getSimpleName(), "Exporting as non-HUDI dataset: " + cfg.targetOutputPath);
- final BaseFileOnlyView fsView = getBaseFileOnlyView(sourceFs, cfg);
+ final BaseFileOnlyView fsView = getBaseFileOnlyView(context, sourceFs, cfg);
Iterator<String> exportingFilePaths = jsc
.parallelize(partitions, partitions.size())
.flatMap(partition -> fsView
@@ -240,8 +240,8 @@ public class HoodieSnapshotExporter {
Config cfg, List<String> partitions, String latestCommitTimestamp,
HoodieTableMetaClient metaClient) throws IOException {
final int parallelism = cfg.parallelism == 0 ? jsc.defaultParallelism() : cfg.parallelism;
- final BaseFileOnlyView fsView = getBaseFileOnlyView(sourceFs, cfg);
final HoodieEngineContext context = new HoodieSparkEngineContext(jsc);
+ final BaseFileOnlyView fsView = getBaseFileOnlyView(context, sourceFs, cfg);
final StorageConfiguration<?> storageConf = context.getStorageConf();
context.setJobStatus(this.getClass().getSimpleName(), "Exporting as HUDI dataset");
List<Pair<String, String>> partitionAndFileList = context.flatMap(partitions, partition -> {
@@ -322,12 +322,12 @@ public class HoodieSnapshotExporter {
}, parallelism);
}
- private BaseFileOnlyView getBaseFileOnlyView(FileSystem sourceFs, Config cfg) {
+ private BaseFileOnlyView getBaseFileOnlyView(HoodieEngineContext engineContext, FileSystem sourceFs, Config cfg) {
HoodieTableMetaClient tableMetadata = HoodieTableMetaClient.builder()
.setConf(HadoopFSUtils.getStorageConfWithCopy(sourceFs.getConf()))
.setBasePath(cfg.sourceBasePath)
.build();
- return new HoodieTableFileSystemView(tableMetadata, tableMetadata
+ return HoodieTableFileSystemView.fileListingBasedFileSystemView(engineContext, tableMetadata, tableMetadata
.getActiveTimeline().getWriteTimeline().filterCompletedInstants());
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
index e9ffcdf4f3b..03af2393137 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java
@@ -65,7 +65,7 @@ import static org.apache.hudi.common.table.HoodieTableMetaClient.reload;
import static org.apache.hudi.common.table.timeline.HoodieInstant.State.REQUESTED;
import static org.apache.hudi.config.HoodieWriteConfig.CLIENT_HEARTBEAT_INTERVAL_IN_MS;
import static org.apache.hudi.config.HoodieWriteConfig.CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES;
-import static org.apache.hudi.metadata.HoodieTableMetadataUtil.getFileSystemView;
+import static org.apache.hudi.metadata.HoodieTableMetadataUtil.getFileSystemViewForMetadataTable;
import static org.apache.hudi.metadata.HoodieTableMetadataUtil.metadataPartitionExists;
import static org.apache.hudi.metadata.MetadataPartitionType.BLOOM_FILTERS;
import static org.apache.hudi.metadata.MetadataPartitionType.COLUMN_STATS;
@@ -435,7 +435,7 @@ public class TestHoodieIndexer extends SparkClientFunctionalTestHarness implemen
.setConf(metaClient.getStorageConf().newInstance()).setBasePath(metaClient.getMetaPath() + "/metadata").build();
List<FileSlice> partitionFileSlices =
HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(
- metadataMetaClient, getFileSystemView(metadataMetaClient), COLUMN_STATS.getPartitionPath());
+ metadataMetaClient, getFileSystemViewForMetadataTable(metadataMetaClient), COLUMN_STATS.getPartitionPath());
assertEquals(partitionFileSlices.size(), colStatsFileGroupCount);
}
@@ -487,7 +487,7 @@ public class TestHoodieIndexer extends SparkClientFunctionalTestHarness implemen
.setConf(metaClient.getStorageConf().newInstance()).setBasePath(metaClient.getMetaPath() + "/metadata").build();
List<FileSlice> partitionFileSlices =
HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(
- metadataMetaClient, getFileSystemView(metadataMetaClient), COLUMN_STATS.getPartitionPath());
+ metadataMetaClient, getFileSystemViewForMetadataTable(metadataMetaClient), COLUMN_STATS.getPartitionPath());
assertEquals(partitionFileSlices.size(), HoodieMetadataConfig.METADATA_INDEX_COLUMN_STATS_FILE_GROUP_COUNT.defaultValue());
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
index fc6d2d027cf..ee62611f1e4 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
@@ -613,8 +613,7 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
Path tempFolderPath = new Path(tempFolder);
// lets move one of the log files from latest file slice to the temp dir. so validation w/ latest file slice should fail.
- HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(
- metaClient, metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(), false);
+ HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context, metaClient, metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(), false);
FileSlice latestFileSlice = fsView.getLatestFileSlices(StringUtils.EMPTY_STRING).filter(fileSlice -> {
return fileSlice.getLogFiles().count() > 0;
}).collect(Collectors.toList()).get(0);
@@ -648,8 +647,7 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
// no exception should be thrown
// let's delete one of the log files from 1st commit and so FS based listing and MDT based listing diverges when all file slices are validated.
- fsView = new HoodieTableFileSystemView(
- metaClient, metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(), false);
+ fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context, metaClient, metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(), false);
HoodieFileGroup fileGroup = fsView.getAllFileGroups(StringUtils.EMPTY_STRING).collect(Collectors.toList()).get(0);
List<FileSlice> allFileSlices = fileGroup.getAllFileSlices().collect(Collectors.toList());
FileSlice earliestFileSlice = allFileSlices.get(allFileSlices.size() - 1);
@@ -1183,7 +1181,7 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
assertFalse(validator.hasValidationFailure());
// lets override one of the latest base file w/ another. so that file slice validation succeeds, but record index comparison fails.
- HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(
+ HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(engineContext,
metaClient, metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(), false);
List<HoodieBaseFile> allBaseFiles = fsView.getLatestBaseFiles(StringUtils.EMPTY_STRING).collect(Collectors.toList());
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index e6331ce0239..947d4475f91 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -80,7 +80,6 @@ import org.apache.hudi.io.hadoop.HoodieAvroParquetReader;
import org.apache.hudi.keygen.ComplexKeyGenerator;
import org.apache.hudi.keygen.NonpartitionedKeyGenerator;
import org.apache.hudi.keygen.SimpleKeyGenerator;
-import org.apache.hudi.metadata.HoodieMetadataFileSystemView;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
import org.apache.hudi.storage.StorageConfiguration;
@@ -2816,7 +2815,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
TestHelpers.assertCommitMetadata("00000", tableBasePath, 1);
} else if (operationType == WriteOperationType.INSERT_OVERWRITE_TABLE) {
HoodieTableMetaClient metaClient = createMetaClient(jsc, tableBasePath);
- final HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline());
+ final HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context, metaClient, metaClient.getCommitsAndCompactionTimeline());
assertEquals(0, fsView.getLatestFileSlices("").count());
TestHelpers.assertCommitMetadata("00000", tableBasePath, 1);
@@ -3111,9 +3110,8 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
new HoodieLocalEngineContext(metaClient.getStorageConf()), metaClient.getStorage(), metaClient.getBasePath(), false);
StorageConfiguration hadoopConf = metaClient.getStorageConf();
HoodieLocalEngineContext engContext = new HoodieLocalEngineContext(hadoopConf);
- HoodieMetadataFileSystemView fsView = new HoodieMetadataFileSystemView(engContext, metaClient,
- metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(),
- HoodieMetadataConfig.newBuilder().enable(false).build());
+ HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(engContext, metaClient,
+ metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants());
List<String> baseFiles = partitions.parallelStream().flatMap(partition -> fsView.getLatestBaseFiles(partition).map(HoodieBaseFile::getPath)).collect(Collectors.toList());
// Verify each partition has one base file because parallelism is 1.
assertEquals(baseFiles.size(), partitions.size());
@@ -3177,7 +3175,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
private Set<String> getAllFileIDsInTable(String tableBasePath, Option<String> partition) {
HoodieTableMetaClient metaClient = createMetaClient(jsc, tableBasePath);
- final HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline());
+ final HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context, metaClient, metaClient.getCommitsAndCompactionTimeline());
Stream<HoodieBaseFile> baseFileStream = partition.isPresent() ? fsView.getLatestBaseFiles(partition.get()) : fsView.getLatestBaseFiles();
return baseFileStream.map(HoodieBaseFile::getFileId).collect(Collectors.toSet());
}