You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vi...@apache.org on 2020/05/18 01:19:00 UTC
[incubator-hudi] branch master updated: [HUDI-895] Remove
unnecessary listing .hoodie folder when using timeline server (#1636)
This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 3c9da2e [HUDI-895] Remove unnecessary listing .hoodie folder when using timeline server (#1636)
3c9da2e is described below
commit 3c9da2e5f038ea90a823334a7b07bb4d13f90996
Author: Balaji Varadarajan <va...@uber.com>
AuthorDate: Sun May 17 18:18:53 2020 -0700
[HUDI-895] Remove unnecessary listing .hoodie folder when using timeline server (#1636)
---
.../org/apache/hudi/client/HoodieWriteClient.java | 6 +--
.../java/org/apache/hudi/table/HoodieTable.java | 6 +--
.../common/table/view/FileSystemViewManager.java | 59 +++++++++++++---------
3 files changed, 40 insertions(+), 31 deletions(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
index c781c69..fa0b15c 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
@@ -540,10 +540,8 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
HoodieTimeline.compareTimestamps(latestPending.getTimestamp(), HoodieTimeline.LESSER_THAN, instantTime),
"Latest pending compaction instant time must be earlier than this instant time. Latest Compaction :"
+ latestPending + ", Ingesting at " + instantTime));
- HoodieTable<T> table = HoodieTable.create(metaClient, config, hadoopConf);
- HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
- String commitActionType = table.getMetaClient().getCommitActionType();
- activeTimeline.createNewInstant(new HoodieInstant(State.REQUESTED, commitActionType, instantTime));
+ metaClient.getActiveTimeline().createNewInstant(new HoodieInstant(State.REQUESTED, metaClient.getCommitActionType(),
+ instantTime));
}
/**
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java
index 8acd351..0584dbf 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -231,21 +231,21 @@ public abstract class HoodieTable<T extends HoodieRecordPayload> implements Seri
* Get the base file only view of the file system for this table.
*/
public BaseFileOnlyView getBaseFileOnlyView() {
- return getViewManager().getFileSystemView(metaClient.getBasePath());
+ return getViewManager().getFileSystemView(metaClient);
}
/**
* Get the full view of the file system for this table.
*/
public SliceView getSliceView() {
- return getViewManager().getFileSystemView(metaClient.getBasePath());
+ return getViewManager().getFileSystemView(metaClient);
}
/**
* Get complete view of the file system for this table with ability to force sync.
*/
public SyncableFileSystemView getHoodieView() {
- return getViewManager().getFileSystemView(metaClient.getBasePath());
+ return getViewManager().getFileSystemView(metaClient);
}
/**
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
index c5e7764..c4ab712 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
@@ -58,10 +58,10 @@ public class FileSystemViewManager {
// Map from Base-Path to View
private final ConcurrentHashMap<String, SyncableFileSystemView> globalViewMap;
// Factory Map to create file-system views
- private final Function2<String, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator;
+ private final Function2<HoodieTableMetaClient, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator;
public FileSystemViewManager(SerializableConfiguration conf, FileSystemViewStorageConfig viewStorageConfig,
- Function2<String, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator) {
+ Function2<HoodieTableMetaClient, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator) {
this.conf = new SerializableConfiguration(conf);
this.viewStorageConfig = viewStorageConfig;
this.globalViewMap = new ConcurrentHashMap<>();
@@ -87,7 +87,21 @@ public class FileSystemViewManager {
* @return
*/
public SyncableFileSystemView getFileSystemView(String basePath) {
- return globalViewMap.computeIfAbsent(basePath, (path) -> viewCreator.apply(path, viewStorageConfig));
+ return globalViewMap.computeIfAbsent(basePath, (path) -> {
+ HoodieTableMetaClient metaClient = new HoodieTableMetaClient(conf.newCopy(), path);
+ return viewCreator.apply(metaClient, viewStorageConfig);
+ });
+ }
+
+ /**
+ * Main API to get the file-system view for the base-path.
+ *
+ * @param metaClient HoodieTableMetaClient
+ * @return
+ */
+ public SyncableFileSystemView getFileSystemView(HoodieTableMetaClient metaClient) {
+ return globalViewMap.computeIfAbsent(metaClient.getBasePath(),
+ (path) -> viewCreator.apply(metaClient, viewStorageConfig));
}
/**
@@ -105,12 +119,11 @@ public class FileSystemViewManager {
*
* @param conf Hadoop Configuration
* @param viewConf View Storage Configuration
- * @param basePath Base Path of table
+ * @param metaClient HoodieTableMetaClient
* @return
*/
private static RocksDbBasedFileSystemView createRocksDBBasedFileSystemView(SerializableConfiguration conf,
- FileSystemViewStorageConfig viewConf, String basePath) {
- HoodieTableMetaClient metaClient = new HoodieTableMetaClient(conf.newCopy(), basePath, true);
+ FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) {
HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
return new RocksDbBasedFileSystemView(metaClient, timeline, viewConf);
}
@@ -120,13 +133,12 @@ public class FileSystemViewManager {
*
* @param conf Hadoop Configuration
* @param viewConf View Storage Configuration
- * @param basePath Base Path of table
+ * @param metaClient HoodieTableMetaClient
* @return
*/
private static SpillableMapBasedFileSystemView createSpillableMapBasedFileSystemView(SerializableConfiguration conf,
- FileSystemViewStorageConfig viewConf, String basePath) {
- LOG.info("Creating SpillableMap based view for basePath " + basePath);
- HoodieTableMetaClient metaClient = new HoodieTableMetaClient(conf.newCopy(), basePath, true);
+ FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) {
+ LOG.info("Creating SpillableMap based view for basePath " + metaClient.getBasePath());
HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
return new SpillableMapBasedFileSystemView(metaClient, timeline, viewConf);
}
@@ -136,13 +148,12 @@ public class FileSystemViewManager {
*
* @param conf Hadoop Configuration
* @param viewConf View Storage Configuration
- * @param basePath Base Path of table
+ * @param metaClient HoodieTableMetaClient
* @return
*/
private static HoodieTableFileSystemView createInMemoryFileSystemView(SerializableConfiguration conf,
- FileSystemViewStorageConfig viewConf, String basePath) {
- LOG.info("Creating InMemory based view for basePath " + basePath);
- HoodieTableMetaClient metaClient = new HoodieTableMetaClient(conf.newCopy(), basePath, true);
+ FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) {
+ LOG.info("Creating InMemory based view for basePath " + metaClient.getBasePath());
HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
return new HoodieTableFileSystemView(metaClient, timeline, viewConf.isIncrementalTimelineSyncEnabled());
}
@@ -177,34 +188,34 @@ public class FileSystemViewManager {
case EMBEDDED_KV_STORE:
LOG.info("Creating embedded rocks-db based Table View");
return new FileSystemViewManager(conf, config,
- (basePath, viewConf) -> createRocksDBBasedFileSystemView(conf, viewConf, basePath));
+ (metaClient, viewConf) -> createRocksDBBasedFileSystemView(conf, viewConf, metaClient));
case SPILLABLE_DISK:
LOG.info("Creating Spillable Disk based Table View");
return new FileSystemViewManager(conf, config,
- (basePath, viewConf) -> createSpillableMapBasedFileSystemView(conf, viewConf, basePath));
+ (metaClient, viewConf) -> createSpillableMapBasedFileSystemView(conf, viewConf, metaClient));
case MEMORY:
LOG.info("Creating in-memory based Table View");
return new FileSystemViewManager(conf, config,
- (basePath, viewConfig) -> createInMemoryFileSystemView(conf, viewConfig, basePath));
+ (metaClient, viewConfig) -> createInMemoryFileSystemView(conf, viewConfig, metaClient));
case REMOTE_ONLY:
LOG.info("Creating remote only table view");
- return new FileSystemViewManager(conf, config, (basePath, viewConfig) -> createRemoteFileSystemView(conf,
- viewConfig, new HoodieTableMetaClient(conf.newCopy(), basePath)));
+ return new FileSystemViewManager(conf, config, (metaClient, viewConfig) -> createRemoteFileSystemView(conf,
+ viewConfig, metaClient));
case REMOTE_FIRST:
LOG.info("Creating remote first table view");
- return new FileSystemViewManager(conf, config, (basePath, viewConfig) -> {
+ return new FileSystemViewManager(conf, config, (metaClient, viewConfig) -> {
RemoteHoodieTableFileSystemView remoteFileSystemView =
- createRemoteFileSystemView(conf, viewConfig, new HoodieTableMetaClient(conf.newCopy(), basePath));
+ createRemoteFileSystemView(conf, viewConfig, metaClient);
SyncableFileSystemView secondaryView;
switch (viewConfig.getSecondaryStorageType()) {
case MEMORY:
- secondaryView = createInMemoryFileSystemView(conf, viewConfig, basePath);
+ secondaryView = createInMemoryFileSystemView(conf, viewConfig, metaClient);
break;
case EMBEDDED_KV_STORE:
- secondaryView = createRocksDBBasedFileSystemView(conf, viewConfig, basePath);
+ secondaryView = createRocksDBBasedFileSystemView(conf, viewConfig, metaClient);
break;
case SPILLABLE_DISK:
- secondaryView = createSpillableMapBasedFileSystemView(conf, viewConfig, basePath);
+ secondaryView = createSpillableMapBasedFileSystemView(conf, viewConfig, metaClient);
break;
default:
throw new IllegalArgumentException("Secondary Storage type can only be in-memory or spillable. Was :"