You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/21 15:57:38 UTC
[4/9] git commit: SLIDER-546 Slider conf dir files are not being
picked up by the AM
SLIDER-546 Slider conf dir files are not being picked up by the AM
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a97af55a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a97af55a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a97af55a
Branch: refs/heads/develop
Commit: a97af55accce5dd59fa6b1f0dfa99f4c69073762
Parents: 7e4abb1
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 21 12:25:44 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 21 12:25:44 2014 +0100
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider | 2 +-
slider-assembly/src/main/scripts/slider.py | 3 +--
.../org/apache/slider/client/SliderClient.java | 12 ++++-----
.../apache/slider/common/tools/SliderUtils.java | 2 +-
.../core/launch/ClasspathConstructor.java | 2 +-
.../server/appmaster/SliderAppMaster.java | 27 ++++++++++++++------
6 files changed, 29 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-assembly/src/main/scripts/slider
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider
index 24ae64e..787016a 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -55,7 +55,7 @@ if [ -f "${confdir}/slider-env.sh" ]; then
. "${confdir}/slider-env.sh"
fi
-slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=${confdir}"
+slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Dslider.confdir=${confdir}"
slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts}
# allow for an extra classpath
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index 2ee172e..94e40a2 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -33,7 +33,7 @@ SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
HADOOP_CONF_DIR = "HADOOP_CONF_DIR"
SLIDER_CLASSNAME = "org.apache.slider.Slider"
-DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s"
+DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Dslider.confdir=%s"
ON_POSIX = 'posix' in sys.builtin_module_names
@@ -211,7 +211,6 @@ def java(classname, args, classpath, jvm_opts_list):
"""
# split the JVM opts by space
# java = "/usr/bin/java"
- prg = "java"
if os.environ["JAVA_HOME"] is not None and os.environ["JAVA_HOME"]:
prg = os.path.join(os.environ["JAVA_HOME"], "bin", "java")
else:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index dc0cab0..5482566 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1146,7 +1146,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// local files or archives as needed
// In this scenario, the jar file for the application master is part of the local resources
Map<String, LocalResource> localResources = amLauncher.getLocalResources();
- // conf directory setup
+
+ // look for the configuration directory named on the command line
Path remoteConfPath = null;
String relativeConfDir = null;
String confdirProp =
@@ -1160,11 +1161,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
confDir);
}
Path localConfDirPath = SliderUtils.createLocalPath(confDir);
- log.debug("Copying AM configuration data from {}", localConfDirPath);
- remoteConfPath = new Path(clusterDirectory,
- SliderKeys.SUBMITTED_CONF_DIR);
- SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath,
- null);
+ log.debug("Slide configuration directory is {}; remote to be {}",
+ localConfDirPath, SliderKeys.SUBMITTED_CONF_DIR);
+ remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR);
+ SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null);
}
// the assumption here is that minimr cluster => this is a test run
// and the classpath can look after itself
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index c034e98..a6d8cd5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1210,10 +1210,10 @@ public final class SliderUtils {
// and nothing else
classpath.appendAll(classpath.localJVMClasspath());
} else {
- classpath.addLibDir(libdir);
if (sliderConfDir != null) {
classpath.addClassDirectory(sliderConfDir);
}
+ classpath.addLibDir(libdir);
classpath.addRemoteClasspathEnvVar();
classpath.appendAll(classpath.yarnApplicationClasspath(config));
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
index 8e49435..f474ffd 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
@@ -113,7 +113,7 @@ public class ClasspathConstructor {
}
public void addClassDirectory(String pathToDir) {
- append(buildLibDir(appendDirectoryTerminator(pathToDir)));
+ append(appendDirectoryTerminator(pathToDir));
}
public void insertClassDirectory(String pathToDir) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a97af55a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index a73fef1..78584c3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -23,8 +23,10 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -78,6 +80,7 @@ import org.apache.slider.api.proto.Messages;
import org.apache.slider.api.proto.SliderClusterAPI;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.SliderAMArgs;
import org.apache.slider.common.params.SliderAMCreateAction;
@@ -375,6 +378,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
public SliderAppMaster() {
super(SERVICE_CLASSNAME_SHORT);
+ new HdfsConfiguration();
+ new YarnConfiguration();
}
/* =================================================================== */
@@ -383,25 +388,31 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@Override //AbstractService
public synchronized void serviceInit(Configuration conf) throws Exception {
-
+ // slider client if found
+
+ Configuration customConf = SliderUtils.loadClientConfigurationResource();
// Load in the server configuration - if it is actually on the Classpath
Configuration serverConf =
ConfigHelper.loadFromResource(SERVER_RESOURCE);
- ConfigHelper.mergeConfigurations(conf, serverConf, SERVER_RESOURCE, true);
+ ConfigHelper.mergeConfigurations(customConf, serverConf, SERVER_RESOURCE, true);
+ serviceArgs.applyDefinitions(customConf);
+ serviceArgs.applyFileSystemBinding(customConf);
+ // conf now contains all customizations
AbstractActionArgs action = serviceArgs.getCoreAction();
SliderAMCreateAction createAction = (SliderAMCreateAction) action;
- //sort out the location of the AM
- serviceArgs.applyDefinitions(conf);
- serviceArgs.applyFileSystemBinding(conf);
+ // sort out the location of the AM
String rmAddress = createAction.getRmAddress();
if (rmAddress != null) {
log.debug("Setting rm address from the command line: {}", rmAddress);
- SliderUtils.setRmSchedulerAddress(conf, rmAddress);
+ SliderUtils.setRmSchedulerAddress(customConf, rmAddress);
}
- serviceArgs.applyDefinitions(conf);
- serviceArgs.applyFileSystemBinding(conf);
+
+ log.info("AM configuration:\n{}",
+ ConfigHelper.dumpConfigToString(customConf));
+
+ ConfigHelper.mergeConfigurations(conf, customConf, CLIENT_RESOURCE, true);
//init security with our conf
if (SliderUtils.isHadoopClusterSecure(conf)) {
log.info("Secure mode with kerberos realm {}",