You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2012/11/19 16:12:24 UTC
svn commit: r1411235 - in
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client:
hadoop-mapreduce-client-app/
hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/
hadoop-mapreduce-client-jobclient/src/test/j...
Author: tomwhite
Date: Mon Nov 19 15:12:22 2012
New Revision: 1411235
URL: http://svn.apache.org/viewvc?rev=1411235&view=rev
Log:
YARN-129. Simplify classpath construction for mini YARN tests.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1411235&r1=1411234&r2=1411235&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml Mon Nov 19 15:12:22 2012
@@ -89,11 +89,6 @@
<phase>test-compile</phase>
</execution>
</executions>
- <configuration>
- <excludes>
- <exclude>mrapp-generated-classpath</exclude>
- </excludes>
- </configuration>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1411235&r1=1411234&r2=1411235&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Mon Nov 19 15:12:22 2012
@@ -18,13 +18,8 @@
package org.apache.hadoop.mapreduce.v2.util;
-import java.io.BufferedReader;
-import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
import java.net.URI;
-import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -134,62 +129,24 @@ public class MRApps extends Apps {
private static void setMRFrameworkClasspath(
Map<String, String> environment, Configuration conf) throws IOException {
- InputStream classpathFileStream = null;
- BufferedReader reader = null;
- try {
- // Get yarn mapreduce-app classpath from generated classpath
- // Works if compile time env is same as runtime. Mainly tests.
- ClassLoader thisClassLoader =
- Thread.currentThread().getContextClassLoader();
- String mrAppGeneratedClasspathFile = "mrapp-generated-classpath";
- classpathFileStream =
- thisClassLoader.getResourceAsStream(mrAppGeneratedClasspathFile);
-
- // Put the file itself on classpath for tasks.
- URL classpathResource = thisClassLoader
- .getResource(mrAppGeneratedClasspathFile);
- if (classpathResource != null) {
- String classpathElement = classpathResource.getFile();
- if (classpathElement.contains("!")) {
- classpathElement = classpathElement.substring(0,
- classpathElement.indexOf("!"));
- } else {
- classpathElement = new File(classpathElement).getParent();
- }
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
- classpathElement);
- }
-
- if (classpathFileStream != null) {
- reader = new BufferedReader(new InputStreamReader(classpathFileStream,
- Charsets.UTF_8));
- String cp = reader.readLine();
- if (cp != null) {
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
- cp.trim());
- }
- }
-
- // Add standard Hadoop classes
- for (String c : conf.getStrings(
- YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
- .trim());
- }
- for (String c : conf.getStrings(
- MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
- MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) {
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
- .trim());
- }
- } finally {
- if (classpathFileStream != null) {
- classpathFileStream.close();
- }
- if (reader != null) {
- reader.close();
- }
+ // Propagate the system classpath when using the mini cluster
+ if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+ Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
+ System.getProperty("java.class.path"));
+ }
+
+ // Add standard Hadoop classes
+ for (String c : conf.getStrings(
+ YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
+ Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
+ .trim());
+ }
+ for (String c : conf.getStrings(
+ MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
+ MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) {
+ Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
+ .trim());
}
// TODO: Remove duplicates.
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java?rev=1411235&r1=1411234&r2=1411235&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java Mon Nov 19 15:12:22 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.RunningJ
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -52,6 +53,8 @@ public class TestEncryptedShuffle {
private static final String BASEDIR =
System.getProperty("test.build.dir", "target/test-dir") + "/" +
TestEncryptedShuffle.class.getSimpleName();
+
+ private String classpathDir;
@BeforeClass
public static void setUp() throws Exception {
@@ -62,27 +65,12 @@ public class TestEncryptedShuffle {
@Before
public void createCustomYarnClasspath() throws Exception {
- String classpathDir =
- KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
-
- URL url = Thread.currentThread().getContextClassLoader().
- getResource("mrapp-generated-classpath");
- File f = new File(url.getPath());
- BufferedReader reader = new BufferedReader(new FileReader(f));
- String cp = reader.readLine();
- cp = cp + ":" + classpathDir;
- f = new File(classpathDir, "mrapp-generated-classpath");
- Writer writer = new FileWriter(f);
- writer.write(cp);
- writer.close();
+ classpathDir = KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
new File(classpathDir, "core-site.xml").delete();
}
@After
public void cleanUpMiniClusterSpecialConfig() throws Exception {
- String classpathDir =
- KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
- new File(classpathDir, "mrapp-generated-classpath").delete();
new File(classpathDir, "core-site.xml").delete();
String keystoresDir = new File(BASEDIR).getAbsolutePath();
KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, classpathDir);
@@ -98,6 +86,9 @@ public class TestEncryptedShuffle {
conf.set("dfs.block.access.token.enable", "false");
conf.set("dfs.permissions", "true");
conf.set("hadoop.security.authentication", "simple");
+ String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH) +
+ File.pathSeparator + classpathDir;
+ conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp);
dfsCluster = new MiniDFSCluster(conf, 1, true, null);
FileSystem fileSystem = dfsCluster.getFileSystem();
fileSystem.mkdirs(new Path("/tmp"));
@@ -113,8 +104,6 @@ public class TestEncryptedShuffle {
mrCluster = MiniMRClientClusterFactory.create(this.getClass(), 1, conf);
// so the minicluster conf is avail to the containers.
- String classpathDir =
- KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
Writer writer = new FileWriter(classpathDir + "/core-site.xml");
mrCluster.getConfig().writeXml(writer);
writer.close();