You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vb...@apache.org on 2020/05/12 16:33:38 UTC
[incubator-hudi] branch master updated: [HUDI-881] Replace part of
spark context by hadoop configuration in AbstractHoodieClient and
HoodieReadClient (#1620)
This is an automated email from the ASF dual-hosted git repository.
vbalaji pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new e8ffc6f [HUDI-881] Replace part of spark context by hadoop configuration in AbstractHoodieClient and HoodieReadClient (#1620)
e8ffc6f is described below
commit e8ffc6f0aa91e2ba2f4affe09d4cb29bd12d41e4
Author: Shen Hong <sh...@126.com>
AuthorDate: Wed May 13 00:33:29 2020 +0800
[HUDI-881] Replace part of spark context by hadoop configuration in AbstractHoodieClient and HoodieReadClient (#1620)
---
.../main/java/org/apache/hudi/client/AbstractHoodieClient.java | 8 ++++----
.../src/main/java/org/apache/hudi/client/HoodieReadClient.java | 4 ++--
.../java/org/apache/hudi/table/action/BaseActionExecutor.java | 4 ++++
.../apache/hudi/table/action/commit/BaseCommitActionExecutor.java | 2 +-
4 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java b/hudi-client/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
index 5e382f1..cdd125e 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
@@ -62,9 +62,9 @@ public abstract class AbstractHoodieClient implements Serializable, AutoCloseabl
protected AbstractHoodieClient(JavaSparkContext jsc, HoodieWriteConfig clientConfig,
Option<EmbeddedTimelineService> timelineServer) {
- this.fs = FSUtils.getFs(clientConfig.getBasePath(), jsc.hadoopConfiguration());
- this.jsc = jsc;
this.hadoopConf = jsc.hadoopConfiguration();
+ this.fs = FSUtils.getFs(clientConfig.getBasePath(), hadoopConf);
+ this.jsc = jsc;
this.basePath = clientConfig.getBasePath();
this.config = clientConfig;
this.timelineServer = timelineServer;
@@ -99,7 +99,7 @@ public abstract class AbstractHoodieClient implements Serializable, AutoCloseabl
if (!timelineServer.isPresent()) {
// Run Embedded Timeline Server
LOG.info("Starting Timeline service !!");
- timelineServer = Option.of(new EmbeddedTimelineService(jsc.hadoopConfiguration(), jsc.getConf(),
+ timelineServer = Option.of(new EmbeddedTimelineService(hadoopConf, jsc.getConf(),
config.getClientSpecifiedViewStorageConfig()));
try {
timelineServer.get().startServer();
@@ -122,6 +122,6 @@ public abstract class AbstractHoodieClient implements Serializable, AutoCloseabl
}
protected HoodieTableMetaClient createMetaClient(boolean loadActiveTimelineOnLoad) {
- return ClientUtils.createMetaClient(jsc.hadoopConfiguration(), config, loadActiveTimelineOnLoad);
+ return ClientUtils.createMetaClient(hadoopConf, config, loadActiveTimelineOnLoad);
}
}
diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java
index b92eaa8..d4249af 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java
@@ -98,7 +98,7 @@ public class HoodieReadClient<T extends HoodieRecordPayload> implements Serializ
this.hadoopConf = jsc.hadoopConfiguration();
final String basePath = clientConfig.getBasePath();
// Create a Hoodie table which encapsulated the commits and files visible
- HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath, true);
+ HoodieTableMetaClient metaClient = new HoodieTableMetaClient(hadoopConf, basePath, true);
this.hoodieTable = HoodieTable.create(metaClient, clientConfig, hadoopConf);
this.index = HoodieIndex.createIndex(clientConfig);
this.sqlContextOpt = Option.empty();
@@ -197,7 +197,7 @@ public class HoodieReadClient<T extends HoodieRecordPayload> implements Serializ
*/
public List<Pair<String, HoodieCompactionPlan>> getPendingCompactions() {
HoodieTableMetaClient metaClient =
- new HoodieTableMetaClient(jsc.hadoopConfiguration(), hoodieTable.getMetaClient().getBasePath(), true);
+ new HoodieTableMetaClient(hadoopConf, hoodieTable.getMetaClient().getBasePath(), true);
return CompactionUtils.getAllPendingCompactionPlans(metaClient).stream()
.map(
instantWorkloadPair -> Pair.of(instantWorkloadPair.getKey().getTimestamp(), instantWorkloadPair.getValue()))
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/action/BaseActionExecutor.java b/hudi-client/src/main/java/org/apache/hudi/table/action/BaseActionExecutor.java
index 5f81db7..9ee4126 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/action/BaseActionExecutor.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/action/BaseActionExecutor.java
@@ -19,6 +19,8 @@
package org.apache.hudi.table.action;
import java.io.Serializable;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
import org.apache.spark.api.java.JavaSparkContext;
@@ -26,6 +28,7 @@ import org.apache.spark.api.java.JavaSparkContext;
public abstract class BaseActionExecutor<R> implements Serializable {
protected final transient JavaSparkContext jsc;
+ protected final transient Configuration hadoopConf;
protected final HoodieWriteConfig config;
@@ -35,6 +38,7 @@ public abstract class BaseActionExecutor<R> implements Serializable {
public BaseActionExecutor(JavaSparkContext jsc, HoodieWriteConfig config, HoodieTable<?> table, String instantTime) {
this.jsc = jsc;
+ this.hadoopConf = jsc.hadoopConfiguration();
this.config = config;
this.table = table;
this.instantTime = instantTime;
diff --git a/hudi-client/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java b/hudi-client/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
index 2daf942..a846de8 100644
--- a/hudi-client/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
+++ b/hudi-client/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
@@ -182,7 +182,7 @@ public abstract class BaseCommitActionExecutor<T extends HoodieRecordPayload<T>>
String actionType = table.getMetaClient().getCommitActionType();
LOG.info("Committing " + instantTime + ", action Type " + actionType);
// Create a Hoodie table which encapsulated the commits and files visible
- HoodieTable<T> table = HoodieTable.create(config, jsc.hadoopConfiguration());
+ HoodieTable<T> table = HoodieTable.create(config, hadoopConf);
HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
HoodieCommitMetadata metadata = new HoodieCommitMetadata();