You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by zi...@apache.org on 2023/09/21 23:21:58 UTC
[gobblin] branch master updated: [GOBBLIN-1914] Configurable Gobblin Application Master class for Yarn (#3781)
This is an automated email from the ASF dual-hosted git repository.
zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 1ca3192a2 [GOBBLIN-1914] Configurable Gobblin Application Master class for Yarn (#3781)
1ca3192a2 is described below
commit 1ca3192a29a3fd45045ca5bcddbdf58763ae1f79
Author: Matthew Ho <mh...@linkedin.com>
AuthorDate: Thu Sep 21 16:21:52 2023 -0700
[GOBBLIN-1914] Configurable Gobblin Application Master class for Yarn (#3781)
* [GOBBLIN-1914] Configurable Gobblin Application Master class for Yarn
* Configurable logs
---
.../gobblin/yarn/GobblinYarnAppLauncher.java | 23 ++++++++++++++++------
.../gobblin/yarn/GobblinYarnConfigurationKeys.java | 3 +++
.../gobblin/yarn/GobblinYarnAppLauncherTest.java | 3 +++
3 files changed, 23 insertions(+), 6 deletions(-)
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
index 48ac89479..391b160fd 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
@@ -801,23 +801,34 @@ public class GobblinYarnAppLauncher {
@VisibleForTesting
protected String buildApplicationMasterCommand(String applicationId, int memoryMbs) {
- String appMasterClassName = GobblinApplicationMaster.class.getSimpleName();
+ Class appMasterClass;
+ try {
+ String appMasterClassName = ConfigUtils.getString(
+ config, GobblinYarnConfigurationKeys.APP_MASTER_CLASS, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_CLASS);
+ appMasterClass = Class.forName(appMasterClassName);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+
+ String logFileName = ConfigUtils.getString(config,
+ GobblinYarnConfigurationKeys.APP_MASTER_LOG_FILE_NAME, appMasterClass.getSimpleName());
+
return new StringBuilder()
.append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
.append(" -Xmx").append((int) (memoryMbs * this.jvmMemoryXmxRatio) - this.jvmMemoryOverheadMbs).append("M")
.append(" -D").append(GobblinYarnConfigurationKeys.JVM_USER_TIMEZONE_CONFIG).append("=").append(this.containerTimezone)
.append(" -D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_DIR_NAME).append("=").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR)
- .append(" -D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=").append(appMasterClassName).append(".").append(ApplicationConstants.STDOUT)
+ .append(" -D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=").append(logFileName).append(".").append(ApplicationConstants.STDOUT)
.append(" ").append(JvmUtils.formatJvmArguments(this.appMasterJvmArgs))
- .append(" ").append(GobblinApplicationMaster.class.getName())
+ .append(" ").append(appMasterClass.getName())
.append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME)
.append(" ").append(this.applicationName)
.append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_ID_OPTION_NAME)
.append(" ").append(applicationId)
.append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append(
- appMasterClassName).append(".").append(ApplicationConstants.STDOUT)
+ logFileName).append(".").append(ApplicationConstants.STDOUT)
.append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append(
- appMasterClassName).append(".").append(ApplicationConstants.STDERR)
+ logFileName).append(".").append(ApplicationConstants.STDERR)
.toString();
}
@@ -1071,4 +1082,4 @@ public class GobblinYarnAppLauncher {
gobblinYarnAppLauncher.launch();
}
-}
\ No newline at end of file
+}
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
index 7088dfa99..f1cc4ada1 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -30,6 +30,9 @@ public class GobblinYarnConfigurationKeys {
public static final String GOBBLIN_YARN_PREFIX = "gobblin.yarn.";
// General Gobblin Yarn application configuration properties.
+ public static final String APP_MASTER_CLASS = GOBBLIN_YARN_PREFIX + "app.master.class";
+ public static final String DEFAULT_APP_MASTER_CLASS = GobblinApplicationMaster.class.getName();
+ public static final String APP_MASTER_LOG_FILE_NAME = GOBBLIN_YARN_PREFIX + "app.master.log.file.name";
public static final String APPLICATION_NAME_KEY = GOBBLIN_YARN_PREFIX + "app.name";
public static final String APP_QUEUE_KEY = GOBBLIN_YARN_PREFIX + "app.queue";
public static final String APP_REPORT_INTERVAL_MINUTES_KEY = GOBBLIN_YARN_PREFIX + "app.report.interval.minutes";
diff --git a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
index 65e9dc269..756e88241 100644
--- a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
+++ b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
@@ -229,6 +229,9 @@ public class GobblinYarnAppLauncherTest implements HelixMessageTestBase {
// 41 is from 64 * 0.8 - 10
Assert.assertTrue(command.contains("-Xmx41"));
+ Assert.assertTrue(command.contains("org.apache.gobblin.yarn.GobblinApplicationMaster"));
+ Assert.assertTrue(command.contains("GobblinApplicationMaster.stdout"));
+ Assert.assertTrue(command.contains("GobblinApplicationMaster.stderr"));
}
@Test