You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by su...@apache.org on 2020/06/05 05:21:40 UTC
[incubator-gobblin] branch master updated: [GOBBLIN-1183] Enable
additional yarn class path set for app master
This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 2dd17fd [GOBBLIN-1183] Enable additional yarn class path set for app master
2dd17fd is described below
commit 2dd17fd8cf66152b98651fe5adccae3d2c84f5d9
Author: Zihan Li <zi...@zihli-mn1.linkedin.biz>
AuthorDate: Thu Jun 4 22:21:33 2020 -0700
[GOBBLIN-1183] Enable additional yarn class path set for app master
Closes #3031 from ZihanLi58/GOBBLIN-1183
---
.../java/org/apache/gobblin/yarn/GobblinApplicationMaster.java | 5 +----
.../java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java | 3 +++
.../org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java | 2 ++
.../src/main/java/org/apache/gobblin/yarn/YarnHelixUtils.java | 10 ++++++++++
4 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java
index 898d96b..5ae3f82 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java
@@ -98,10 +98,7 @@ public class GobblinApplicationMaster extends GobblinClusterManager {
this.applicationLauncher
.addService(logCopier);
}
- if (!ConfigUtils.emptyIfNotPresent(config, GobblinYarnConfigurationKeys.GOBBLIN_YARN_ADDITIONAL_CLASSPATHS).equals(StringUtils.EMPTY)){
- LOGGER.info("set additional yarn class path to yarnConfig");
- yarnConfiguration.setStrings(GobblinYarnConfigurationKeys.GOBBLIN_YARN_ADDITIONAL_CLASSPATHS, config.getString(GobblinYarnConfigurationKeys.GOBBLIN_YARN_ADDITIONAL_CLASSPATHS));
- }
+ YarnHelixUtils.setAdditionalYarnClassPath(config, yarnConfiguration);
this.yarnService = buildYarnService(this.config, applicationName, this.applicationId, yarnConfiguration, this.fs);
this.applicationLauncher.addService(this.yarnService);
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
index 38d4c38..3215a90 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
@@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.avro.Schema;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.reflect.ConstructorUtils;
import org.apache.commons.mail.EmailException;
import org.apache.hadoop.conf.Configuration;
@@ -251,6 +252,7 @@ public class GobblinYarnAppLauncher {
InstanceType.SPECTATOR, zkConnectionString);
this.yarnConfiguration = yarnConfiguration;
+ YarnHelixUtils.setAdditionalYarnClassPath(config, this.yarnConfiguration);
this.yarnConfiguration.set("fs.automatic.close", "false");
this.yarnClient = YarnClient.createYarnClient();
this.yarnClient.init(this.yarnConfiguration);
@@ -601,6 +603,7 @@ public class GobblinYarnAppLauncher {
YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication();
ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext();
appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE);
+ appSubmissionContext.setMaxAppAttempts(ConfigUtils.getInt(config, GobblinYarnConfigurationKeys.APP_MASTER_MAX_ATTEMPTS_KEY, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_MAX_ATTEMPTS_KEY));
ApplicationId applicationId = appSubmissionContext.getApplicationId();
GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse();
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
index 7132386..1a4bbc3 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -47,6 +47,8 @@ public class GobblinYarnConfigurationKeys {
public static final String APP_MASTER_WORK_DIR_NAME = "appmaster";
public static final String APP_MASTER_JVM_ARGS_KEY = GOBBLIN_YARN_PREFIX + "app.master.jvm.args";
public static final String APP_MASTER_SERVICE_CLASSES = GOBBLIN_YARN_PREFIX + "app.master.serviceClasses";
+ public static final String APP_MASTER_MAX_ATTEMPTS_KEY = GOBBLIN_YARN_PREFIX + "app.master.max.attempts";
+ public static final int DEFAULT_APP_MASTER_MAX_ATTEMPTS_KEY = 10;
// Amount of overhead to subtract when computing the Xmx value. This is to account for non-heap memory, like metaspace
// and stack memory
public static final String APP_MASTER_JVM_MEMORY_OVERHEAD_MBS_KEY = GOBBLIN_YARN_PREFIX + "app.master.jvmMemoryOverheadMbs";
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnHelixUtils.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnHelixUtils.java
index 0b58aea..177e177 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnHelixUtils.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnHelixUtils.java
@@ -17,11 +17,14 @@
package org.apache.gobblin.yarn;
+import com.typesafe.config.Config;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.gobblin.util.ConfigUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -138,6 +141,13 @@ public class YarnHelixUtils {
return environmentVariableMap;
}
+ public static void setAdditionalYarnClassPath(Config config, Configuration yarnConfiguration) {
+ if (!ConfigUtils.emptyIfNotPresent(config, GobblinYarnConfigurationKeys.GOBBLIN_YARN_ADDITIONAL_CLASSPATHS).equals(
+ StringUtils.EMPTY)){
+ yarnConfiguration.setStrings(GobblinYarnConfigurationKeys.GOBBLIN_YARN_ADDITIONAL_CLASSPATHS, config.getString(GobblinYarnConfigurationKeys.GOBBLIN_YARN_ADDITIONAL_CLASSPATHS));
+ }
+ }
+
/**
* Return the identifier of the containerId. The identifier is the substring in the containerId representing
* the sequential number of the container.