You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by go...@apache.org on 2015/05/29 08:57:54 UTC
incubator-slider git commit: SLIDER-891 Add ability to set Slider AM
launch environment during cluster create/start
Repository: incubator-slider
Updated Branches:
refs/heads/develop dae41ee06 -> ed7b52eab
SLIDER-891 Add ability to set Slider AM launch environment during cluster create/start
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ed7b52ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ed7b52ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ed7b52ea
Branch: refs/heads/develop
Commit: ed7b52eab76ba74ab8a7ab274aab1d83d5eea5ef
Parents: dae41ee
Author: Gour Saha <go...@apache.org>
Authored: Thu May 28 23:15:26 2015 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Thu May 28 23:15:26 2015 -0700
----------------------------------------------------------------------
slider-assembly/src/conf-hdp/slider-client.xml | 10 ++
.../org/apache/slider/client/SliderClient.java | 65 ++++++++-
.../apache/slider/common/SliderXmlConfKeys.java | 1 +
.../apache/slider/common/tools/SliderUtils.java | 9 +-
.../server/appmaster/SliderAppMaster.java | 3 +
.../client/TestSliderClientMethods.groovy | 143 +++++++++++++++++++
6 files changed, 229 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ed7b52ea/slider-assembly/src/conf-hdp/slider-client.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/conf-hdp/slider-client.xml b/slider-assembly/src/conf-hdp/slider-client.xml
index 136abf0..ef2496b 100644
--- a/slider-assembly/src/conf-hdp/slider-client.xml
+++ b/slider-assembly/src/conf-hdp/slider-client.xml
@@ -37,6 +37,11 @@
</property>
<property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
<name>slider.security.protocol.acl</name>
<value>*</value>
<description>When security is enabled, set appropriate acl. Default value means allow everyone.</description>
@@ -78,4 +83,9 @@
</property>
-->
+ <property>
+ <name>slider.am.launch.env</name>
+ <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ed7b52ea/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 98f1344..33e8b07 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import com.google.common.io.Files;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -48,6 +49,7 @@ import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -183,6 +185,7 @@ import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -192,6 +195,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.apache.hadoop.registry.client.binding.RegistryUtils.ServiceRecordMarshal;
@@ -2204,6 +2208,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
amLauncher.setEnv("LANG", "en_US.UTF-8");
amLauncher.setEnv("LC_ALL", "en_US.UTF-8");
amLauncher.setEnv("LANGUAGE", "en_US.UTF-8");
+ amLauncher.putEnv(getAmLaunchEnv(config));
+
+ for (Map.Entry<String, String> envs : SliderUtils.getSystemEnv().entrySet()) {
+ log.debug("System env {}={}", envs.getKey(), envs.getValue());
+ }
if (log.isDebugEnabled()) {
log.debug("AM classpath={}", classpath);
log.debug("Environment Map:\n{}",
@@ -2318,6 +2327,60 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return launchedApplication;
}
+ protected Map<String, String> getAmLaunchEnv(Configuration config) {
+ String sliderAmLaunchEnv = config.get(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV);
+ log.debug("{} = {}", SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv);
+ // Multiple env variables can be specified with a comma (,) separator
+ String[] envs = StringUtils.isEmpty(sliderAmLaunchEnv) ? null
+ : sliderAmLaunchEnv.split(",");
+ if (ArrayUtils.isEmpty(envs)) {
+ return Collections.emptyMap();
+ }
+ Map<String, String> amLaunchEnv = new HashMap<String, String>();
+ for (String env : envs) {
+ if (StringUtils.isNotEmpty(env)) {
+ // Each env name/value is separated by equals sign (=)
+ String[] tokens = env.split("=");
+ if (tokens != null && tokens.length == 2) {
+ String envKey = tokens[0];
+ String envValue = tokens[1];
+ for (Map.Entry<String, String> placeholder : generatePlaceholderKeyValueMap(
+ env).entrySet()) {
+ if (StringUtils.isNotEmpty(placeholder.getValue())) {
+ envValue = envValue.replaceAll(
+ Pattern.quote(placeholder.getKey()), placeholder.getValue());
+ }
+ }
+ if (Shell.WINDOWS) {
+ envValue = "%" + envKey + "%;" + envValue;
+ } else {
+ envValue = "$" + envKey + ":" + envValue;
+ }
+ log.info("Setting AM launch env {}={}", envKey, envValue);
+ amLaunchEnv.put(envKey, envValue);
+ }
+ }
+ }
+ return amLaunchEnv;
+ }
+
+ protected Map<String, String> generatePlaceholderKeyValueMap(String env) {
+ String PLACEHOLDER_PATTERN = "\\$\\{[^{]+\\}";
+ Pattern placeholderPattern = Pattern.compile(PLACEHOLDER_PATTERN);
+ Matcher placeholderMatcher = placeholderPattern.matcher(env);
+ Map<String, String> placeholderKeyValueMap = new HashMap<String, String>();
+ if (placeholderMatcher.find()) {
+ String placeholderKey = placeholderMatcher.group();
+ String systemKey = placeholderKey
+ .substring(2, placeholderKey.length() - 1).toUpperCase()
+ .replaceAll("\\.", "_");
+ String placeholderValue = SliderUtils.getSystemEnv(systemKey);
+ log.debug("Placeholder {}={}", placeholderKey, placeholderValue);
+ placeholderKeyValueMap.put(placeholderKey, placeholderValue);
+ }
+ return placeholderKeyValueMap;
+ }
+
private void propagatePythonExecutable(Configuration config,
AggregateConf instanceDefinition) {
String pythonExec = config.get(
@@ -3866,7 +3929,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// verbose?
if (diagnosticArgs.verbose) {
// do the environment
- Map<String, String> env = System.getenv();
+ Map<String, String> env = SliderUtils.getSystemEnv();
Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet());
StringBuilder builder = new StringBuilder("Environment variables:\n");
for (String key : envList) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ed7b52ea/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index b337a1a..4e9d8c6 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -173,4 +173,5 @@ public interface SliderXmlConfKeys {
public static final String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
"10000,6,60000,10"; //t1,n1,t2,n2,...
+ String KEY_AM_LAUNCH_ENV = "slider.am.launch.env";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ed7b52ea/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index c06aa4a..057e61a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -2360,5 +2360,12 @@ public final class SliderUtils {
public static int compareTwoLongsReverse(long x, long y) {
return (x < y) ? +1 : ((x == y) ? 0 : -1);
}
-
+
+ public static String getSystemEnv(String property) {
+ return System.getenv(property);
+ }
+
+ public static Map<String, String> getSystemEnv() {
+ return System.getenv();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ed7b52ea/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7da627e..019ec71 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -457,6 +457,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
log.info("AM configuration:\n{}",
ConfigHelper.dumpConfigToString(customConf));
+ for (Map.Entry<String, String> envs : System.getenv().entrySet()) {
+ log.info("System env {}={}", envs.getKey(), envs.getValue());
+ }
ConfigHelper.mergeConfigurations(conf, customConf, SLIDER_CLIENT_XML, true);
//init security with our conf
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ed7b52ea/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy
new file mode 100644
index 0000000..168415c
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client
+
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.util.Shell
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.exceptions.YarnException
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.build.InstanceBuilder
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.launch.LaunchedApplication
+import org.apache.slider.core.main.ServiceLauncherBaseTest
+import org.apache.slider.core.persist.LockAcquireFailedException
+import org.easymock.EasyMock
+import org.junit.Assert
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.powermock.api.easymock.PowerMock
+import org.powermock.core.classloader.annotations.PrepareForTest
+import org.powermock.modules.junit4.PowerMockRunner
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(SliderUtils.class)
+class TestSliderClientMethods extends ServiceLauncherBaseTest {
+ String AM_ENV = "LD_LIBRARY_PATH"
+ String PLACEHOLDER_KEY = "\${distro.version}"
+ String PLACEHOLDER_SYSTEM_KEY = "DISTRO_VERSION"
+ String PLACEHOLDER_VALUE = "1.0.0"
+ String AM_ENV_2 = "PATH"
+ String PLACEHOLDER_KEY_2 = "\${native.version}"
+ String PLACEHOLDER_SYSTEM_KEY_2 = "NATIVE_VERSION"
+ String PLACEHOLDER_VALUE_2 = "2.0.0"
+
+ @Test
+ public void testGeneratePlaceholderKeyValueMap() throws Throwable {
+ TestSliderClient testSliderClient = new TestSliderClient()
+
+ PowerMock.mockStatic(System.class);
+ EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY))
+ .andReturn(PLACEHOLDER_VALUE).anyTimes()
+ PowerMock.replayAll()
+
+ Map<String, String> placeholders = testSliderClient.generatePlaceholderKeyValueMap(
+ AM_ENV + "=/usr/lib/" + PLACEHOLDER_KEY)
+ Assert.assertTrue(placeholders.containsKey(PLACEHOLDER_KEY))
+ Assert.assertEquals("Should be equal", PLACEHOLDER_VALUE,
+ placeholders.get(PLACEHOLDER_KEY))
+
+ PowerMock.verifyAll()
+ println("Placeholders = " + placeholders)
+ }
+
+ @Test
+ public void testSetAmLaunchEnv() throws Throwable {
+ TestSliderClient testSliderClient = new TestSliderClient()
+ YarnConfiguration conf = SliderUtils.createConfiguration()
+ conf.set(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, AM_ENV + "=/usr/lib/"
+ + PLACEHOLDER_KEY)
+
+ PowerMock.mockStatic(System.class);
+ EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY))
+ .andReturn(PLACEHOLDER_VALUE)
+ PowerMock.replayAll()
+
+ Map<String, String> amLaunchEnv = testSliderClient.getAmLaunchEnv(conf)
+ Assert.assertNotNull(amLaunchEnv)
+ Assert.assertNotNull(amLaunchEnv.get(AM_ENV))
+ Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV),
+ (Shell.WINDOWS ? "%" + AM_ENV + "%;" : "\$" + AM_ENV + ":") +
+ "/usr/lib/" + PLACEHOLDER_VALUE)
+
+ PowerMock.verifyAll()
+ println("amLaunchEnv = " + amLaunchEnv)
+ }
+
+ @Test
+ public void testSetAmLaunchEnvMulti() throws Throwable {
+ TestSliderClient testSliderClient = new TestSliderClient()
+ YarnConfiguration conf = SliderUtils.createConfiguration()
+ conf.set(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, AM_ENV + "=/usr/lib/"
+ + PLACEHOLDER_KEY + "," + AM_ENV_2 + "=/usr/bin/" + PLACEHOLDER_KEY_2)
+
+ PowerMock.mockStatic(System.class);
+ EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY))
+ .andReturn(PLACEHOLDER_VALUE)
+ EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY_2))
+ .andReturn(PLACEHOLDER_VALUE_2)
+ PowerMock.replayAll()
+
+ Map<String, String> amLaunchEnv = testSliderClient.getAmLaunchEnv(conf)
+ Assert.assertNotNull(amLaunchEnv)
+ Assert.assertEquals("Should have 2 envs", amLaunchEnv.size(), 2)
+ Assert.assertNotNull(amLaunchEnv.get(AM_ENV))
+ Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV),
+ (Shell.WINDOWS ? "%" + AM_ENV + "%;" : "\$" + AM_ENV + ":") +
+ "/usr/lib/" + PLACEHOLDER_VALUE)
+ Assert.assertNotNull(amLaunchEnv.get(AM_ENV_2))
+ Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV_2),
+ (Shell.WINDOWS ? "%" + AM_ENV_2 + "%;" : "\$" + AM_ENV_2 + ":") +
+ "/usr/bin/" + PLACEHOLDER_VALUE_2)
+
+ PowerMock.verifyAll()
+ println("amLaunchEnv = " + amLaunchEnv)
+ }
+
+ static class TestSliderClient extends SliderClient {
+ @Override
+ protected void persistInstanceDefinition(boolean overwrite,
+ Path appconfdir,
+ InstanceBuilder builder)
+ throws IOException, SliderException, LockAcquireFailedException {
+ super.persistInstanceDefinition(overwrite, appconfdir, builder)
+ }
+
+ @Override
+ LaunchedApplication launchApplication(String clustername,
+ Path clusterDirectory,
+ AggregateConf instanceDefinition,
+ boolean debugAM)
+ throws YarnException, IOException {
+ return new LaunchedApplication(clustername, new SliderYarnClientImpl());
+ }
+ }
+}