You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/10/25 16:31:52 UTC
git commit: Revert "SLIDER-515. Ensure Slider clients uploads all
jars needed by AppMaster and construct own class path"
Repository: incubator-slider
Updated Branches:
refs/heads/develop fbeb02e64 -> 06ff4c2ea
Revert "SLIDER-515. Ensure Slider clients uploads all jars needed by AppMaster and construct own class path"
This reverts commit eb790e780d262eb29b5551d21956faac78347145.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/06ff4c2e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/06ff4c2e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/06ff4c2e
Branch: refs/heads/develop
Commit: 06ff4c2ea51acb44f308a0ce52b5a50e85442bfa
Parents: fbeb02e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Oct 25 07:31:18 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Oct 25 07:31:18 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider | 4 +-
slider-assembly/src/main/scripts/slider.py | 2 -
.../org/apache/slider/common/SliderKeys.java | 6 ---
.../apache/slider/common/tools/SliderUtils.java | 40 +-------------------
.../apache/slider/providers/ProviderUtils.java | 28 --------------
.../slideram/SliderAMClientProvider.java | 37 ++++++++++++++----
6 files changed, 31 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-assembly/src/main/scripts/slider
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider
index 80b5f32..4228843 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -56,9 +56,7 @@ if [ -f "${confdir}/slider-env.sh" ]; then
fi
slider_confdir_opts="-Dslider.confdir=${confdir}"
-libdir_jvm_opts="-Dslider.libdir=${libdir}"
-
-slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true ${libdir_jvm_opts} -Xmx256m"
+slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m"
slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts}
# allow for an extra classpath
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index c72fc0d..a6dd90e 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -247,8 +247,6 @@ def main():
executeEnvSh(confdir)
jvm_opts_list = (SLIDER_CONFDIR_OPTS % confdir).split()
default_jvm_opts = DEFAULT_JVM_OPTS
- libdir_jvm_opts = "-Dslider.libdir={0}".format(libdir)
- default_jvm_opts = "{0} {1}".format(default_jvm_opts, libdir_jvm_opts)
slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts)
jvm_opts_list.extend(slider_jvm_opts.split())
slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 048dfa7..ae58ef3 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -122,12 +122,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
String PROPERTY_CONF_DIR = "slider.confdir";
/**
- * JVM property to define the slider lib directory;
- * this is set by the slider script: {@value}
- */
- String PROPERTY_LIB_DIR = "slider.libdir";
-
- /**
* name of generated dir for this conf: {@value}
*/
String SUBMITTED_CONF_DIR = "confdir";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 43dfbd0..33e028f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -71,7 +70,6 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
-import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
@@ -1172,42 +1170,6 @@ public final class SliderUtils {
return res;
}
- /**
- * Submit a JAR containing and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param libDir lib directory
- * @param srcPath copy jars from
- * @throws IOException, SliderException trouble copying to HDFS
- */
- public static void putAllJars(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libDir,
- String srcPath
- )
- throws IOException, SliderException {
- log.info("Loading all dependencies from {}", srcPath);
- if (SliderUtils.isSet(srcPath)) {
- File srcFolder = new File(srcPath);
- FilenameFilter jarFilter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- String lowercaseName = name.toLowerCase();
- if (lowercaseName.endsWith(".jar")) {
- return true;
- } else {
- return false;
- }
- }
- };
- File[] listOfJars = srcFolder.listFiles(jarFilter);
- for (File jarFile : listOfJars) {
- LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
- providerResources.put(libDir + "/" + jarFile.getName(), res);
- }
- }
- }
-
public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
Map<String, Map<String, String>> dest =
new HashMap<String, Map<String, String>>();
@@ -1339,7 +1301,7 @@ public final class SliderUtils {
}
classpath.addLibDir(libdir);
classpath.addRemoteClasspathEnvVar();
- classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$());
+ classpath.appendAll(classpath.yarnApplicationClasspath(config));
}
return classpath;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index e26105e..8e77a9c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -175,34 +175,6 @@ public class ProviderUtils implements RoleKeys {
}
/**
- * Loads all dependency jars from the default path
- * @param providerResources map of provider resources to add these entries to
- * @param sliderFileSystem target filesystem
- * @param tempPath path in the cluster FS for temp files
- * @param libDir relative directory to place resources
- * @param libLocalSrcDir explicitly supplied local libs dir
- * @throws IOException
- * @throws SliderException
- */
- public static void addAllDependencyJars(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libDir,
- String libLocalSrcDir
- ) throws
- IOException,
- SliderException {
- String libSrcToUse = libLocalSrcDir;
- if (SliderUtils.isSet(libLocalSrcDir)) {
- File file = new File(libLocalSrcDir);
- if (!file.exists() || !file.isDirectory()) {
- throw new BadCommandArgumentsException("Supplied lib src dir %s is not valid", libLocalSrcDir);
- }
- }
- SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath, libDir, libSrcToUse);
- }
-
- /**
* build the log directory
* @return the log dir
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06ff4c2e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index 60edce7..5edf1bf 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -18,6 +18,11 @@
package org.apache.slider.providers.slideram;
+import com.beust.jcommander.JCommander;
+import com.codahale.metrics.MetricRegistry;
+import com.google.gson.GsonBuilder;
+import org.apache.curator.CuratorZookeeperClient;
+import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
@@ -42,6 +47,7 @@ import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
+import org.mortbay.jetty.security.SslSelectChannelConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -181,14 +187,29 @@ public class SliderAMClientProvider extends AbstractClientProvider
libdir,
miniClusterTestRun);
- String libDirProp =
- System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
- log.info("Loading all dependencies for AM.");
- ProviderUtils.addAllDependencyJars(providerResources,
- fileSystem,
- tempPath,
- libdir,
- libDirProp);
+ Class<?>[] classes = {
+ JCommander.class,
+ GsonBuilder.class,
+ SslSelectChannelConnector.class,
+
+ CuratorFramework.class,
+ CuratorZookeeperClient.class,
+ MetricRegistry.class
+ };
+ String[] jars =
+ {
+ JCOMMANDER_JAR,
+ GSON_JAR,
+ "jetty-sslengine.jar",
+
+ "curator-framework.jar",
+ "curator-client.jar",
+ "metrics-core.jar"
+ };
+ ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath,
+ libdir, jars,
+ classes);
+
addKeytabResourceIfNecessary(fileSystem,
launcher,
instanceDescription,