You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/10/19 04:46:17 UTC
svn commit: r1399953 - in /hadoop/common/branches/branch-1-win: ./
src/core/org/apache/hadoop/fs/ src/mapred/org/apache/hadoop/mapred/
src/test/org/apache/hadoop/mapred/ src/test/testshell/
Author: suresh
Date: Fri Oct 19 02:46:16 2012
New Revision: 1399953
URL: http://svn.apache.org/viewvc?rev=1399953&view=rev
Log:
HADOOP-8899. Classpath exceeds maximum Windows command limit. Contributed by Ahmed El Baz.
Modified:
hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/DefaultTaskController.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JvmManager.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskController.java
hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java
hadoop/common/branches/branch-1-win/src/test/testshell/ExternalMapReduce.java
Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Fri Oct 19 02:46:16 2012
@@ -172,3 +172,6 @@ Branch-hadoop-1-win - unreleased
HDFS-4065. TestDFSShell.testGet sporadically fails attempting to corrupt
block files due to race condition. (Chris Nauroth via suresh)
+
+ HADOOP-8899. Classpath exceeds maximum Windows command limit.
+ (Ahmed El Baz via suresh)
Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java Fri Oct 19 02:46:16 2012
@@ -19,7 +19,12 @@
package org.apache.hadoop.fs;
import java.io.*;
+import java.net.URL;
import java.util.Enumeration;
+import java.util.List;
+import java.util.jar.Attributes;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.apache.commons.logging.Log;
@@ -818,4 +823,37 @@ public class FileUtil {
}
return fileNames;
}
+
+ /**
+ * Create a JAR file under at the given path, referencing all entries in the classpath list.
+ * @param jarFile file to create with classpath entries in its manifest
+ * @param classPaths entries to be added in the manifest of the created Jar
+ * @return jarFile created with classpath entries
+ */
+ public static File createJarWithClassPath(File jarFile, List<String> classPaths)
+ throws IOException {
+ StringBuffer jarClsPath = new StringBuffer();
+
+ // Append all entries in classpath list to the Jar
+ for (String clsEntry : classPaths) {
+ URL fileUrl = new URL(new File(clsEntry).toURI().toString());
+ jarClsPath.append(fileUrl.toExternalForm());
+ jarClsPath.append(" ");
+ }
+
+ // Create the manifest
+ Manifest jarManifest = new Manifest();
+ jarManifest.getMainAttributes().putValue(
+ Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
+
+ jarManifest.getMainAttributes().putValue(
+ Attributes.Name.CLASS_PATH.toString(), jarClsPath.toString().trim());
+
+ // Write the manifest to output JAR file
+ JarOutputStream jarStream = new JarOutputStream(
+ new FileOutputStream(jarFile), jarManifest);
+
+ jarStream.close();
+ return jarFile;
+ }
}
Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/DefaultTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/DefaultTaskController.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/DefaultTaskController.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/DefaultTaskController.java Fri Oct 19 02:46:16 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapreduce.serve
import org.apache.hadoop.mapred.TaskTracker.LocalStorage;
import org.apache.hadoop.util.ProcessTree.Signal;
import org.apache.hadoop.util.ProcessTree;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.commons.logging.Log;
@@ -55,6 +56,8 @@ public class DefaultTaskController exten
private static final Log LOG =
LogFactory.getLog(DefaultTaskController.class);
+ // Prefix of the Jar file containing the classpath
+ protected static final String JAR_CLASSPATH_PREFIX = "classpath-";
private FileSystem fs;
@Override
public void setConf(Configuration conf) {
@@ -88,6 +91,38 @@ public class DefaultTaskController exten
File currentWorkDirectory,
String stdout,
String stderr) throws IOException {
+ // Assume the classpath is already given in setup or jvmArguments
+ return launchTask(user, jobId, attemptId, setup, jvmArguments, null,
+ currentWorkDirectory, stdout, stderr);
+ }
+
+ /**
+ * Create all of the directories for the task and launches the child jvm.
+ * Uses all items in the classpaths list as the classpath for the task to start
+ * If classPaths is not provided, then it is assumed it is already set as one
+ * of the setup steps, or in jvmArguments
+ * @param user the user name
+ * @param jobId the jobId in question
+ * @param attemptId the attempt id (cleanup attempts have .cleanup suffix)
+ * @param setup list of shell commands to execute before the jvm
+ * @param jvmArguments list of jvm arguments
+ * @param classpaths list of classpath items
+ * @param currentWorkDirectory the full path of the cwd for the task
+ * @param stdout the file to redirect stdout to
+ * @param stderr the file to redirect stderr to
+ * @return the exit code for the task
+ * @throws IOException
+ */
+ @Override
+ public int launchTask(String user,
+ String jobId,
+ String attemptId,
+ List<String> setup,
+ List<String> jvmArguments,
+ List<String> classPaths,
+ File currentWorkDirectory,
+ String stdout,
+ String stderr) throws IOException {
ShellCommandExecutor shExec = null;
try {
FileSystem localFs = FileSystem.getLocal(getConf());
@@ -109,6 +144,30 @@ public class DefaultTaskController exten
throw new IOException("Mkdirs failed to create "
+ logLocation);
}
+
+ //If provided, use all items in the classpaths list as the classpath for
+ //the task to start
+ if (classPaths != null && classPaths.size() > 0) {
+ String clsPaths;
+
+ if (Shell.WINDOWS) {
+ // Prepare the path of the Jar file to create
+ File jarFile = File.createTempFile(JAR_CLASSPATH_PREFIX, ".jar",
+ currentWorkDirectory.getParentFile());
+
+ // Create a Jar file referencing all entries in the classPaths list.
+ clsPaths = FileUtil.createJarWithClassPath(jarFile, classPaths).
+ getCanonicalPath();
+ } else {
+ // Get the classpath string
+ clsPaths = StringUtils.join(File.pathSeparator, classPaths);
+ }
+
+ // Add the classpath as the first Java argument
+ jvmArguments.add(1, "-classpath");
+ jvmArguments.add(2, clsPaths);
+ }
+
//read the configuration for the job
FileSystem rawFs = FileSystem.getLocal(getConf()).getRaw();
long logSize = 0; //TODO MAPREDUCE-1100
Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JvmManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JvmManager.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JvmManager.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/JvmManager.java Fri Oct 19 02:46:16 2012
@@ -52,9 +52,16 @@ class JvmManager {
private JvmManagerForType reduceJvmManager;
public JvmEnv constructJvmEnv(List<String> setup, Vector<String>vargs,
+ List<String> classPathsList, File stdout, File stderr, long logSize,
+ File workDir, JobConf conf) {
+ return new JvmEnv(setup,vargs,classPathsList,stdout,stderr,logSize,
+ workDir,conf);
+ }
+
+ public JvmEnv constructJvmEnv(List<String> setup, Vector<String>vargs,
File stdout,File stderr,long logSize, File workDir,
JobConf conf) {
- return new JvmEnv(setup,vargs,stdout,stderr,logSize,workDir,conf);
+ return constructJvmEnv(setup,vargs,null,stdout,stderr,logSize,workDir,conf);
}
public JvmManager(TaskTracker tracker) {
@@ -497,7 +504,7 @@ class JvmManager {
taskAttemptId.toString();
exitCode = tracker.getTaskController().launchTask(user,
jvmId.jobId.toString(), taskAttemptIdStr, env.setup,
- env.vargs, env.workDir, env.stdout.toString(),
+ env.vargs, env.classpaths, env.workDir, env.stdout.toString(),
env.stderr.toString());
}
} catch (IOException ioe) {
@@ -592,6 +599,7 @@ class JvmManager {
static class JvmEnv { //Helper class
List<String> vargs;
List<String> setup;
+ List<String> classpaths;
File stdout;
File stderr;
File workDir;
@@ -601,12 +609,19 @@ class JvmManager {
public JvmEnv(List <String> setup, Vector<String> vargs, File stdout,
File stderr, long logSize, File workDir, JobConf conf) {
+ this(setup, vargs, null, stdout, stderr, logSize, workDir, conf);
+ }
+
+ // Construct JvmEnv with classpaths to use
+ public JvmEnv(List <String> setup, Vector<String> vargs, List <String> classpaths,
+ File stdout, File stderr, long logSize, File workDir, JobConf conf) {
this.setup = setup;
this.vargs = vargs;
this.stdout = stdout;
this.stderr = stderr;
this.workDir = workDir;
this.conf = conf;
+ this.classpaths = classpaths;
}
}
}
Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java Fri Oct 19 02:46:16 2012
@@ -204,9 +204,31 @@ class LinuxTaskController extends TaskCo
File currentWorkDirectory,
String stdout,
String stderr) throws IOException {
+ return launchTask(user, jobId, attemptId, setup, jvmArguments,
+ null, currentWorkDirectory, stdout, stderr);
+ }
+ @Override
+ public int launchTask(String user,
+ String jobId,
+ String attemptId,
+ List<String> setup,
+ List<String> jvmArguments,
+ List<String> classPaths,
+ File currentWorkDirectory,
+ String stdout,
+ String stderr) throws IOException {
ShellCommandExecutor shExec = null;
try {
+ if (classPaths != null && classPaths.size() > 0) {
+ // Get the classpath string
+ String clsPaths = StringUtils.join(File.pathSeparator, classPaths);
+
+ // Add the classpath as the first Java argument
+ jvmArguments.add(1, "-classpath");
+ jvmArguments.add(2, clsPaths);
+ }
+
FileSystem rawFs = FileSystem.getLocal(getConf()).getRaw();
long logSize = 0; //TODO MAPREDUCE-1100
// get the JVM command line.
@@ -259,7 +281,7 @@ class LinuxTaskController extends TaskCo
}
return 0;
}
-
+
@Override
public void createLogDir(TaskAttemptID taskID,
boolean isCleanup) throws IOException {
Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskController.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskController.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskController.java Fri Oct 19 02:46:16 2012
@@ -63,7 +63,7 @@ public abstract class TaskController imp
final public static FsPermission TASK_LAUNCH_SCRIPT_PERMISSION =
FsPermission.createImmutable((short) 0700); // rwx--------
-
+
public Configuration getConf() {
return conf;
}
@@ -127,6 +127,34 @@ public abstract class TaskController imp
String stderr) throws IOException;
/**
+ * Create all of the directories for the task and launches the child jvm.
+ * Uses all items in the classpaths list as the classpath for the task to start
+ * If classPaths is not provided, then it is assumed it is already set as one
+ * of the setup steps, or in jvmArguments
+ * @param user the user name
+ * @param jobId the jobId in question
+ * @param attemptId the attempt id (cleanup attempts have .cleanup suffix)
+ * @param setup list of shell commands to execute before the jvm
+ * @param jvmArguments list of jvm arguments
+ * @param classpaths list of classpath items
+ * @param currentWorkDirectory the full path of the cwd for the task
+ * @param stdout the file to redirect stdout to
+ * @param stderr the file to redirect stderr to
+ * @return the exit code for the task
+ * @throws IOException
+ */
+ public abstract
+ int launchTask(String user,
+ String jobId,
+ String attemptId,
+ List<String> setup,
+ List<String> jvmArguments,
+ List<String> classPaths,
+ File currentWorkDirectory,
+ String stdout,
+ String stderr) throws IOException;
+
+ /**
* Send a signal to a task pid as the user.
* @param user the user name
* @param taskPid the pid of the task
Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/TaskRunner.java Fri Oct 19 02:46:16 2012
@@ -78,8 +78,6 @@ abstract class TaskRunner extends Thread
static final String HADOOP_WORK_DIR = "HADOOP_WORK_DIR";
- static final String JAVA_CLASSPATH = "CLASSPATH";
-
static final String MAPRED_ADMIN_USER_ENV =
"mapreduce.admin.user.env";
@@ -219,8 +217,6 @@ abstract class TaskRunner extends Thread
// Accumulates class paths for child.
List<String> classPathsList = getClassPaths(conf, workDir,
taskDistributedCacheManager);
- String classPaths = StringUtils.join(SYSTEM_PATH_SEPARATOR,
- classPathsList);
long logSize = TaskLog.getTaskLogLength(conf);
@@ -239,8 +235,8 @@ abstract class TaskRunner extends Thread
stderr);
Map<String, String> env = new HashMap<String, String>();
- errorInfo = getVMEnvironment(errorInfo, user, workDir, classPaths,
- conf, env, taskid, logSize);
+ errorInfo = getVMEnvironment(errorInfo, user, workDir, conf, env,
+ taskid, logSize);
// flatten the env as a set of export commands
List <String> setupCmds = new ArrayList<String>();
@@ -255,7 +251,7 @@ abstract class TaskRunner extends Thread
}
setupCmds.add(setup);
- launchJvmAndWait(setupCmds, vargs, stdout, stderr, logSize, workDir);
+ launchJvmAndWait(setupCmds, vargs, classPathsList, stdout, stderr, logSize, workDir);
tracker.getTaskTrackerInstrumentation().reportTaskEnd(t.getTaskID());
if (exitCodeSet) {
if (!killed && exitCode != 0) {
@@ -293,11 +289,12 @@ abstract class TaskRunner extends Thread
}
}
- void launchJvmAndWait(List <String> setup, Vector<String> vargs, File stdout,
- File stderr, long logSize, File workDir)
+ void launchJvmAndWait(List <String> setup, Vector<String> vargs,
+ List<String> classPathsList, File stdout, File stderr, long logSize,
+ File workDir)
throws InterruptedException, IOException {
- jvmManager.launchJvm(this, jvmManager.constructJvmEnv(setup, vargs, stdout,
- stderr, logSize, workDir, conf));
+ jvmManager.launchJvm(this, jvmManager.constructJvmEnv(setup, vargs, classPathsList,
+ stdout, stderr, logSize, workDir, conf));
synchronized (lock) {
while (!done) {
lock.wait();
@@ -305,6 +302,12 @@ abstract class TaskRunner extends Thread
}
}
+ void launchJvmAndWait(List <String> setup, Vector<String> vargs, File stdout,
+ File stderr, long logSize, File workDir)
+ throws InterruptedException, IOException {
+ launchJvmAndWait(setup, vargs, null, stdout, stderr, logSize, workDir);
+ }
+
/**
* Prepare the log files for the task
*
@@ -568,9 +571,8 @@ abstract class TaskRunner extends Thread
}
private String getVMEnvironment(String errorInfo, String user, File workDir,
- String classPaths, JobConf conf,
- Map<String, String> env, TaskAttemptID taskid,
- long logSize
+ JobConf conf, Map<String, String> env,
+ TaskAttemptID taskid, long logSize
) throws Throwable {
StringBuffer ldLibraryPath = new StringBuffer();
ldLibraryPath.append(workDir.toString());
@@ -582,7 +584,6 @@ abstract class TaskRunner extends Thread
}
env.put("LD_LIBRARY_PATH", ldLibraryPath.toString());
env.put(HADOOP_WORK_DIR, workDir.toString());
- env.put(JAVA_CLASSPATH, classPaths.toString());
//update user configured login-shell properties
updateUserLoginEnv(errorInfo, user, conf, env);
// put jobTokenFile name into env
Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java (original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestJvmManager.java Fri Oct 19 02:46:16 2012
@@ -22,8 +22,12 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Vector;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.TrackerDistributedCacheManager;
import org.apache.hadoop.fs.FileUtil;
@@ -45,6 +49,7 @@ import org.junit.Before;
import org.junit.Test;
public class TestJvmManager {
+ private static final Log LOG = LogFactory.getLog(TestJvmManager.class);
private static File TEST_DIR = new File(System.getProperty("test.build.data",
"/tmp"), TestJvmManager.class.getSimpleName());
private static int MAP_SLOTS = 1;
@@ -110,6 +115,14 @@ public class TestJvmManager {
return script;
}
+ // Create an empty shell script to run from tasks.
+ private File writeEmptyScript(String fileName) throws IOException {
+ File script = new File(TEST_DIR, fileName);
+ script.createNewFile();
+ script.setExecutable(true);
+ return script;
+ }
+
/**
* Tests the jvm kill from JvmRunner and JvmManager simultaneously.
*
@@ -232,4 +245,152 @@ public class TestJvmManager {
private void setThreadCaughtException() {
threadCaughtException = true;
}
+
+ /**
+ * Test launchJvmAndWait overloads, and verify performance difference does
+ * not exceed TIME_DIFF_THRESHOLD
+ *
+ * More details:
+ * - Create a Task that will simply invoke an empty script
+ * - Run the task using the default launchJvmAndWait (RUN_JVM_COUNT) times
+ * and measure total time
+ * - Run the task using the overload of launchJvmAndWait which takes a list
+ * of classpath entries (RUN_JVM_COUNT) times, and measure total time
+ * - Assert the time difference is not more than TIME_DIFF_THRESHOLD
+ */
+ @Test
+ public void testJvmLaunchWithClasspathPerf() throws Exception {
+
+ final int RUN_JVM_COUNT = 200;
+ final int TIME_DIFF_THRESHOLD = 10;
+
+ String jvmTaskCmdName = Shell.WINDOWS ? "writeToFile.cmd" : "writeToFile";
+ final Vector<String> vargs = new Vector<String>(2);
+ vargs.add(writeEmptyScript(jvmTaskCmdName).getAbsolutePath());
+
+ final File workDir = new File(TEST_DIR, "work");
+ final File stdout = new File(TEST_DIR, "stdout");
+ final File stderr = new File(TEST_DIR, "stderr");
+
+ // Ensure all files are deleted from previous tests
+ if (workDir.exists()) {
+ FileUtil.fullyDelete(workDir);
+ }
+ if (stdout.exists()) {
+ stdout.delete();
+ }
+ if (stderr.exists()) {
+ stderr.delete();
+ }
+
+ TaskRunner taskRunner;
+
+ // Create a class-path list
+ List<String> classPaths = new ArrayList<String>();
+ for (int clsPathElemnts = 0; clsPathElemnts < 10; ++clsPathElemnts) {
+ classPaths.add(TEST_DIR.getPath());
+ classPaths.add(workDir.getPath());
+ classPaths.add(stdout.getPath());
+ classPaths.add(stderr.getPath());
+ }
+
+ // Get the start time before launching the tasks
+ long startTime = 0;
+ long endTime = 0;
+ long totalTimeNoJar = 0;
+ long totalTimeWithJar = 0;
+
+ // Test launching the the task without constructing classpath jar
+ for (int iNoJar = 0; iNoJar < RUN_JVM_COUNT; ++iNoJar) {
+ // Create a new task and name it using the current run count
+ taskRunner = prepareNewTask(0, iNoJar, 0);
+
+ // Vargs are changed by the below overload of launchJvmAndWait which add
+ // the classpath as args, so we use a copy of vargs with every iteration
+ Vector<String> vargsCopy = new Vector<String>(vargs);
+
+ // Get start time
+ startTime = System.currentTimeMillis();
+
+ // Launch the the task without constructing classpath jar
+ taskRunner.launchJvmAndWait(null, vargsCopy, null, stdout, stderr,
+ 100, workDir);
+
+ // Get end time
+ endTime = System.currentTimeMillis();
+
+ totalTimeNoJar = totalTimeNoJar + (endTime - startTime);
+
+ // Clean generated files
+ workDir.delete();
+ stdout.delete();
+ stderr.delete();
+ }
+
+ // Test launching the the task with constructing classpath jar
+ for (int iWithJar = 0; iWithJar < RUN_JVM_COUNT; ++iWithJar) {
+ // Create a new task and name it using the current run count
+ taskRunner = prepareNewTask(0, iWithJar, 1);
+
+ // Vargs are changed by the below overload of launchJvmAndWait which add
+ // the classpath as args, so we use a copy of vargs with every iteration
+ Vector<String> vargsCopy = new Vector<String>(vargs);
+
+ // Get start time
+ startTime = System.currentTimeMillis();
+
+ // Launch the the task constructing classpath jar
+ taskRunner.launchJvmAndWait(null, vargsCopy, classPaths, stdout, stderr,
+ 100, workDir);
+
+ // Get end time
+ endTime = System.currentTimeMillis();
+
+ totalTimeWithJar = totalTimeWithJar + (endTime - startTime);
+
+ // Clean generated files
+ workDir.delete();
+ stdout.delete();
+ stderr.delete();
+ }
+
+ // Measure the time difference
+ double timeDiffMilli = Math.abs(totalTimeWithJar - totalTimeNoJar);
+ double diffPercentage = (timeDiffMilli / totalTimeNoJar) * 100;
+
+ // Log results
+ LOG.info(
+ "Time taken for launchJvmAndWait without classpath (milli seconds): "
+ + totalTimeNoJar);
+ LOG.info(
+ "Time taken for launchJvmAndWait with classpath (milli seconds): "
+ + totalTimeWithJar);
+ LOG.info("Time difference is: " + diffPercentage + "%");
+
+ // Verify difference does not exceed TIME_DIFF_THRESHOLD
+ assertTrue(diffPercentage <= TIME_DIFF_THRESHOLD);
+ }
+
+ /**
+ * Helper function to create a new task with the given jobId, ttaskId,
+ * and attemptId
+ * Returns the TaskRunner used to launch the task
+ */
+ private TaskRunner prepareNewTask(int jobId, int ttaskId, int attemptId)
+ throws IOException{
+ JobConf taskConf = new JobConf(ttConf);
+ TaskAttemptID attemptID = new TaskAttemptID("test", jobId, true, ttaskId,
+ attemptId);
+ Task task = new MapTask(null, attemptID, 0, null, MAP_SLOTS);
+ task.setUser(user);
+ task.setConf(taskConf);
+ TaskInProgress tip = tt.new TaskInProgress(task, taskConf);
+ RunningJob rjob = new RunningJob(attemptID.getJobID());
+ TaskController taskController = new DefaultTaskController();
+ taskController.setConf(ttConf);
+ rjob.distCacheMgr =
+ new TrackerDistributedCacheManager(ttConf, taskController).
+ newTaskDistributedCacheManager(attemptID.getJobID(), taskConf);
+ return task.createRunner(tt, tip, rjob);
+ }
}
Modified: hadoop/common/branches/branch-1-win/src/test/testshell/ExternalMapReduce.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/testshell/ExternalMapReduce.java?rev=1399953&r1=1399952&r2=1399953&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/testshell/ExternalMapReduce.java (original)
+++ hadoop/common/branches/branch-1-win/src/test/testshell/ExternalMapReduce.java Fri Oct 19 02:46:16 2012
@@ -19,8 +19,13 @@
package testshell;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Iterator;
+import java.util.jar.Attributes;
+import java.util.jar.JarInputStream;
+import java.util.jar.Manifest;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
@@ -101,6 +106,44 @@ public class ExternalMapReduce extends C
}
}
+ // Check if the given item exists in classpath
+ private static boolean itemInClasspath(String elementName)
+ throws FileNotFoundException, IOException {
+ boolean found = false;
+ String classpath = System.getProperty("java.class.path");
+
+ // Check if the element exists in the classpath
+ if (classpath.indexOf(elementName) >= 0) {
+ found = true;
+ } else {
+ // In case of Windows, classpath is embedded in a referencing jar
+ if (Shell.WINDOWS) {
+ String[] cpFiles = classpath.split(File.pathSeparator);
+ if (cpFiles != null && cpFiles.length == 1 &&
+ cpFiles[0].endsWith(".jar")) {
+ // Search for the element in the jar manifest
+ found = itemInJarClasspath(elementName, cpFiles[0]);
+ }
+ }
+ }
+ return found;
+ }
+
+ // Check if the given item exists as a classpath element in a jar file
+ private static boolean itemInJarClasspath(String elementName,
+ String jarFileName)
+ throws FileNotFoundException, IOException{
+ // Load the Jar manifest
+ JarInputStream jarStream = new JarInputStream(
+ new FileInputStream(jarFileName));
+ Manifest jarManifest = jarStream.getManifest();
+ String classpath = jarManifest.getMainAttributes().getValue(
+ Attributes.Name.CLASS_PATH.toString());
+
+ // Check for the element in the classpath list
+ return (classpath.indexOf(elementName) >= 0);
+ }
+
public static class MapClass extends MapReduceBase
implements Mapper<WritableComparable, Writable,
WritableComparable, IntWritable> {
@@ -110,10 +153,10 @@ public class ExternalMapReduce extends C
throws IOException {
//check for classpath
String classpath = System.getProperty("java.class.path");
- if (classpath.indexOf("testjob.jar") == -1) {
+ if (!itemInClasspath("testjob.jar")) {
throw new IOException("failed to find in the library " + classpath);
}
- if (classpath.indexOf("test.jar") == -1) {
+ if (!itemInClasspath("test.jar")) {
throw new IOException("failed to find the library test.jar in"
+ classpath);
}