You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2019/05/20 14:53:22 UTC
[hadoop] branch trunk updated: YARN-9546. Add configuration option
for YARN Native services AM classpath. Contributed by Gergely Pollak.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 24c53e0 YARN-9546. Add configuration option for YARN Native services AM classpath. Contributed by Gergely Pollak.
24c53e0 is described below
commit 24c53e057a237f78e1433d724df5ffe7961579a5
Author: Sunil G <su...@apache.org>
AuthorDate: Mon May 20 10:53:01 2019 -0400
YARN-9546. Add configuration option for YARN Native services AM classpath. Contributed by Gergely Pollak.
---
.../hadoop/yarn/service/client/ServiceClient.java | 12 +++++++----
.../hadoop/yarn/service/conf/YarnServiceConf.java | 2 ++
.../hadoop/yarn/service/utils/ServiceUtils.java | 7 +++++++
.../yarn/service/client/TestServiceClient.java | 23 ++++++++++++++++++++++
.../src/main/resources/yarn-default.xml | 10 ++++++++++
5 files changed, 50 insertions(+), 4 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 08352a8..030e0db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -1234,11 +1234,15 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return cmdStr;
}
- private Map<String, String> addAMEnv() throws IOException {
+ @VisibleForTesting
+ protected Map<String, String> addAMEnv() throws IOException {
Map<String, String> env = new HashMap<>();
- ClasspathConstructor classpath =
- buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
- .getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
+ ClasspathConstructor classpath = buildClasspath(
+ YarnServiceConstants.SUBMITTED_CONF_DIR,
+ "lib",
+ fs,
+ getConfig().get(YarnServiceConf.YARN_SERVICE_CLASSPATH, ""),
+ getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
env.put("CLASSPATH", classpath.buildClasspath());
env.put("LANG", "en_US.UTF-8");
env.put("LC_ALL", "en_US.UTF-8");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index b9568f2..58fe70b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -60,6 +60,8 @@ public class YarnServiceConf {
public static final String ROLLING_LOG_INCLUSION_PATTERN = "yarn.service.rolling-log.include-pattern";
public static final String ROLLING_LOG_EXCLUSION_PATTERN = "yarn.service.rolling-log.exclude-pattern";
+ public static final String YARN_SERVICE_CLASSPATH = "yarn.service.classpath";
+
public static final String YARN_SERVICES_SYSTEM_SERVICE_DIRECTORY =
YARN_SERVICE_PREFIX + "system-service.dir";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
index a76b64d..34d2ba3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
@@ -451,6 +451,7 @@ public final class ServiceUtils {
* @param sliderConfDir relative path to the dir containing slider config
* options to put on the classpath -or null
* @param libdir directory containing the JAR files
+ * @param configClassPath extra class path configured in yarn-site.xml
* @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
* (and hence the current classpath should be used, not anything built up)
* @return a classpath
@@ -458,6 +459,7 @@ public final class ServiceUtils {
public static ClasspathConstructor buildClasspath(String sliderConfDir,
String libdir,
SliderFileSystem sliderFileSystem,
+ String configClassPath,
boolean usingMiniMRCluster) {
ClasspathConstructor classpath = new ClasspathConstructor();
@@ -479,6 +481,11 @@ public final class ServiceUtils {
classpath.addRemoteClasspathEnvVar();
classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
}
+
+ if (!configClassPath.isEmpty()) {
+ classpath.appendAll(Arrays.asList(configClassPath.split(",")));
+ }
+
return classpath;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java
index 4527da4..c66c4ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java
@@ -77,6 +77,29 @@ public class TestServiceClient {
new ServiceTestUtils.ServiceFSWatcher();
@Test
+ public void testAMEnvCustomClasspath() throws Exception {
+ Service service = createService();
+ service.getComponents().forEach(comp ->
+ comp.setRestartPolicy(Component.RestartPolicyEnum.NEVER));
+ ServiceClient client = MockServiceClient.create(rule, service, true);
+ //saving the original value of the param, for restoration purposes
+ String oldParam = client.getConfig().get("yarn.service.classpath", "");
+ String originalPath = client.addAMEnv().get("CLASSPATH");
+
+ client.getConfig().set("yarn.service.classpath", "{{VAR_1}},{{VAR_2}}");
+ String newPath = client.addAMEnv().get("CLASSPATH");
+
+ Assert.assertEquals(originalPath + "<CPS>{{VAR_1}}<CPS>{{VAR_2}}", newPath);
+ //restoring the original value for service classpath
+ client.getConfig().set("yarn.service.classpath", oldParam);
+
+ newPath = client.addAMEnv().get("CLASSPATH");
+ Assert.assertEquals(originalPath, newPath);
+
+ client.stop();
+ }
+
+ @Test
public void testUpgradeDisabledByDefault() throws Exception {
Service service = createService();
ServiceClient client = MockServiceClient.create(rule, service, false);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 9741f6c..87c2f13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -4211,4 +4211,14 @@
<name>yarn.resourcemanager.activities-manager.app-activities.max-queue-length</name>
<value>1000</value>
</property>
+
+ <property>
+ <description>
+ Comma separated extra class path parameters for yarn services AM.
+ These path elements will be appended to the end of the YARN service AM
+ classpath.
+ </description>
+ <name>yarn.service.classpath</name>
+ <value></value>
+ </property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org