You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/10/25 00:35:57 UTC
git commit: SLIDER-515. Ensure Slider clients uploads all jars needed
by AppMaster and construct own class path
Repository: incubator-slider
Updated Branches:
refs/heads/develop a8d31ee27 -> eb790e780
SLIDER-515. Ensure Slider clients uploads all jars needed by AppMaster and construct own class path
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eb790e78
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eb790e78
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eb790e78
Branch: refs/heads/develop
Commit: eb790e780d262eb29b5551d21956faac78347145
Parents: a8d31ee
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Oct 24 15:35:18 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Oct 24 15:35:18 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider | 4 +-
slider-assembly/src/main/scripts/slider.py | 2 +
.../org/apache/slider/common/SliderKeys.java | 6 +++
.../apache/slider/common/tools/SliderUtils.java | 40 +++++++++++++++++++-
.../apache/slider/providers/ProviderUtils.java | 28 ++++++++++++++
.../slideram/SliderAMClientProvider.java | 37 ++++--------------
6 files changed, 86 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-assembly/src/main/scripts/slider
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider
index 4228843..80b5f32 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -56,7 +56,9 @@ if [ -f "${confdir}/slider-env.sh" ]; then
fi
slider_confdir_opts="-Dslider.confdir=${confdir}"
-slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m"
+libdir_jvm_opts="-Dslider.libdir=${libdir}"
+
+slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true ${libdir_jvm_opts} -Xmx256m"
slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts}
# allow for an extra classpath
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index a6dd90e..c72fc0d 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -247,6 +247,8 @@ def main():
executeEnvSh(confdir)
jvm_opts_list = (SLIDER_CONFDIR_OPTS % confdir).split()
default_jvm_opts = DEFAULT_JVM_OPTS
+ libdir_jvm_opts = "-Dslider.libdir={0}".format(libdir)
+ default_jvm_opts = "{0} {1}".format(default_jvm_opts, libdir_jvm_opts)
slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts)
jvm_opts_list.extend(slider_jvm_opts.split())
slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index ae58ef3..048dfa7 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -122,6 +122,12 @@ public interface SliderKeys extends SliderXmlConfKeys {
String PROPERTY_CONF_DIR = "slider.confdir";
/**
+ * JVM property to define the slider lib directory;
+ * this is set by the slider script: {@value}
+ */
+ String PROPERTY_LIB_DIR = "slider.libdir";
+
+ /**
* name of generated dir for this conf: {@value}
*/
String SUBMITTED_CONF_DIR = "confdir";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 33e028f..43dfbd0 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -70,6 +71,7 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
+import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
@@ -1170,6 +1172,42 @@ public final class SliderUtils {
return res;
}
+ /**
+ * Submit a JAR containing and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param libDir lib directory
+ * @param srcPath copy jars from
+ * @throws IOException, SliderException trouble copying to HDFS
+ */
+ public static void putAllJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String srcPath
+ )
+ throws IOException, SliderException {
+ log.info("Loading all dependencies from {}", srcPath);
+ if (SliderUtils.isSet(srcPath)) {
+ File srcFolder = new File(srcPath);
+ FilenameFilter jarFilter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ String lowercaseName = name.toLowerCase();
+ if (lowercaseName.endsWith(".jar")) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ };
+ File[] listOfJars = srcFolder.listFiles(jarFilter);
+ for (File jarFile : listOfJars) {
+ LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
+ providerResources.put(libDir + "/" + jarFile.getName(), res);
+ }
+ }
+ }
+
public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
Map<String, Map<String, String>> dest =
new HashMap<String, Map<String, String>>();
@@ -1301,7 +1339,7 @@ public final class SliderUtils {
}
classpath.addLibDir(libdir);
classpath.addRemoteClasspathEnvVar();
- classpath.appendAll(classpath.yarnApplicationClasspath(config));
+ classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$());
}
return classpath;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index 8e77a9c..e26105e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -175,6 +175,34 @@ public class ProviderUtils implements RoleKeys {
}
/**
+ * Loads all dependency jars from the default path
+ * @param providerResources map of provider resources to add these entries to
+ * @param sliderFileSystem target filesystem
+ * @param tempPath path in the cluster FS for temp files
+ * @param libDir relative directory to place resources
+ * @param libLocalSrcDir explicitly supplied local libs dir
+ * @throws IOException
+ * @throws SliderException
+ */
+ public static void addAllDependencyJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String libLocalSrcDir
+ ) throws
+ IOException,
+ SliderException {
+ String libSrcToUse = libLocalSrcDir;
+ if (SliderUtils.isSet(libLocalSrcDir)) {
+ File file = new File(libLocalSrcDir);
+ if (!file.exists() || !file.isDirectory()) {
+ throw new BadCommandArgumentsException("Supplied lib src dir %s is not valid", libLocalSrcDir);
+ }
+ }
+ SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath, libDir, libSrcToUse);
+ }
+
+ /**
* build the log directory
* @return the log dir
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb790e78/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index 5edf1bf..60edce7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -18,11 +18,6 @@
package org.apache.slider.providers.slideram;
-import com.beust.jcommander.JCommander;
-import com.codahale.metrics.MetricRegistry;
-import com.google.gson.GsonBuilder;
-import org.apache.curator.CuratorZookeeperClient;
-import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
@@ -47,7 +42,6 @@ import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
-import org.mortbay.jetty.security.SslSelectChannelConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -187,29 +181,14 @@ public class SliderAMClientProvider extends AbstractClientProvider
libdir,
miniClusterTestRun);
- Class<?>[] classes = {
- JCommander.class,
- GsonBuilder.class,
- SslSelectChannelConnector.class,
-
- CuratorFramework.class,
- CuratorZookeeperClient.class,
- MetricRegistry.class
- };
- String[] jars =
- {
- JCOMMANDER_JAR,
- GSON_JAR,
- "jetty-sslengine.jar",
-
- "curator-framework.jar",
- "curator-client.jar",
- "metrics-core.jar"
- };
- ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath,
- libdir, jars,
- classes);
-
+ String libDirProp =
+ System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
+ log.info("Loading all dependencies for AM.");
+ ProviderUtils.addAllDependencyJars(providerResources,
+ fileSystem,
+ tempPath,
+ libdir,
+ libDirProp);
addKeytabResourceIfNecessary(fileSystem,
launcher,
instanceDescription,