You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by xi...@apache.org on 2019/10/14 16:19:55 UTC
[samza] branch master updated: SAMZA-2160 - Ability to configure
application type on YARN deploys (#1185)
This is an automated email from the ASF dual-hosted git repository.
xinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git
The following commit(s) were added to refs/heads/master by this push:
new ebca883 SAMZA-2160 - Ability to configure application type on YARN deploys (#1185)
ebca883 is described below
commit ebca8837e09e514f9fbf75ddb11b7e3c376ab066
Author: Abhishek Shivanna <as...@linkedin.com>
AuthorDate: Mon Oct 14 09:19:44 2019 -0700
SAMZA-2160 - Ability to configure application type on YARN deploys (#1185)
---
.../src/main/java/org/apache/samza/config/YarnConfig.java | 12 ++++++++++++
.../main/scala/org/apache/samza/job/yarn/ClientHelper.scala | 4 +---
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/samza-yarn/src/main/java/org/apache/samza/config/YarnConfig.java b/samza-yarn/src/main/java/org/apache/samza/config/YarnConfig.java
index 58adf26..4b37a81 100644
--- a/samza-yarn/src/main/java/org/apache/samza/config/YarnConfig.java
+++ b/samza-yarn/src/main/java/org/apache/samza/config/YarnConfig.java
@@ -32,6 +32,14 @@ public class YarnConfig extends MapConfig {
public static final String PACKAGE_PATH = "yarn.package.path";
/**
+ * Type of Samza application submitted to YARN
+ * Can be used to distinguish between different job types/APIs (eg: HighLevel, LowLevel, SQL, Beam etc)
+ */
+ public static final String YARN_APPLICATION_TYPE = "yarn.application.type";
+ public static final String DEFAULT_SAMZA_APPLICATION_TYPE = "Samza";
+
+
+ /**
* Name of YARN queue to run jobs on
*/
public static final String QUEUE_NAME = "yarn.queue";
@@ -219,6 +227,10 @@ public class YarnConfig extends MapConfig {
return get(YARN_APPLICATION_MODIFY_ACL, null);
}
+ public String getYarnApplicationType() {
+ return get(YARN_APPLICATION_TYPE, DEFAULT_SAMZA_APPLICATION_TYPE);
+ }
+
/**
* Helper function to get all application acls
* @return a map of {@link ApplicationAccessType} to {@link String} for all the acls defined
diff --git a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/ClientHelper.scala b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/ClientHelper.scala
index da4d836..196ac91 100644
--- a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/ClientHelper.scala
+++ b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/ClientHelper.scala
@@ -60,8 +60,6 @@ import org.apache.http.impl.client.HttpClientBuilder
import org.apache.samza.webapp.ApplicationMasterRestClient
object ClientHelper {
- val applicationType = "Samza"
-
val CREDENTIALS_FILE = "credentials"
val SOURCE = "yarn"
@@ -230,7 +228,7 @@ class ClientHelper(conf: Configuration) extends Logging {
}
appCtx.setAMContainerSpec(containerCtx)
- appCtx.setApplicationType(ClientHelper.applicationType)
+ appCtx.setApplicationType(yarnConfig.getYarnApplicationType)
info("submitting application request for %s" format appId.get)
yarnClient.submitApplication(appCtx)
appId