You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2008/09/16 14:47:39 UTC
svn commit: r695844 - in /hadoop/core/trunk: ./
src/core/org/apache/hadoop/util/ src/test/org/apache/hadoop/mapred/
src/test/testshell/
Author: ddas
Date: Tue Sep 16 05:47:39 2008
New Revision: 695844
URL: http://svn.apache.org/viewvc?rev=695844&view=rev
Log:
HADOOP-3570. Includes user specified libjar files in the client side classpath path. Contributed by Sharad Agarwal.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java
hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java
hadoop/core/trunk/src/test/testshell/ExternalMapReduce.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=695844&r1=695843&r2=695844&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Sep 16 05:47:39 2008
@@ -559,6 +559,9 @@
HADOOP-4099. Fix null pointer when using HFTP from an 0.18 server.
(dhruba via omalley)
+ HADOOP-3570. Includes user specified libjar files in the client side
+ classpath path. (Sharad Agarwal via ddas)
+
Release 0.18.1 - 2008-09-17
IMPROVEMENTS
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java?rev=695844&r1=695843&r2=695844&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java Tue Sep 16 05:47:39 2008
@@ -21,6 +21,8 @@
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
+import java.net.URL;
+import java.net.URLClassLoader;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -228,6 +230,11 @@
if (line.hasOption("libjars")) {
conf.set("tmpjars",
validateFiles(line.getOptionValue("libjars"), conf));
+ //setting libjars in client classpath
+ ClassLoader loader = new URLClassLoader(getLibJars(conf),
+ conf.getClassLoader());
+ Thread.currentThread().setContextClassLoader(loader);
+ conf.setClassLoader(loader);
}
if (line.hasOption("files")) {
conf.set("tmpfiles",
@@ -249,6 +256,26 @@
}
conf.setBoolean("mapred.used.genericoptionsparser", true);
}
+
+ /**
+ * If libjars are set in the conf, parse the libjars.
+ * @param conf
+ * @return libjar urls
+ * @throws IOException
+ */
+ public static URL[] getLibJars(Configuration conf) throws IOException {
+ String jars = conf.get("tmpjars");
+ if(jars==null) {
+ return null;
+ }
+ String[] files = jars.split(",");
+ URL[] cp = new URL[files.length];
+ for (int i=0;i<cp.length;i++) {
+ Path tmp = new Path(files[i]);
+ cp[i] = FileSystem.getLocal(conf).pathToFile(tmp).toURI().toURL();
+ }
+ return cp;
+ }
/**
* takes input as a comma separated list of files
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java?rev=695844&r1=695843&r2=695844&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java Tue Sep 16 05:47:39 2008
@@ -27,6 +27,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.mapred.JobClient;
/** Run a Hadoop job jar. */
public class RunJar {
@@ -141,6 +142,15 @@
classPath.add(libs[i].toURL());
}
}
+
+ //adding libjars to the classpath
+ Configuration conf = JobClient.getCommandLineConfig();
+ URL[] libJars = GenericOptionsParser.getLibJars(conf);
+ if(libJars!=null) {
+ for(URL url : libJars){
+ classPath.add(url);
+ }
+ }
ClassLoader loader =
new URLClassLoader(classPath.toArray(new URL[0]));
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java?rev=695844&r1=695843&r2=695844&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java Tue Sep 16 05:47:39 2008
@@ -68,8 +68,15 @@
args[3] = "build/test/testjar/testjob.jar";
args[4] = input.toString();
args[5] = output.toString();
- int ret = ToolRunner.run(mr.createJobConf(),
+
+ JobConf jobConf = mr.createJobConf();
+ //before running the job, verify that libjar is not in client classpath
+ assertTrue("libjar not in client classpath", loadLibJar(jobConf)==null);
+ int ret = ToolRunner.run(jobConf,
new testshell.ExternalMapReduce(), args);
+ //after running the job, verify that libjar is in the client classpath
+ assertTrue("libjar added to client classpath", loadLibJar(jobConf)!=null);
+
assertTrue("not failed ", ret != -1);
f.delete();
thisbuildDir.delete();
@@ -78,4 +85,13 @@
if (mr != null) {mr.shutdown();};
}
}
+
+ @SuppressWarnings("unchecked")
+ private Class loadLibJar(JobConf jobConf) {
+ try {
+ return jobConf.getClassByName("testjar.ClassWordCount");
+ } catch (ClassNotFoundException e) {
+ return null;
+ }
+ }
}
Modified: hadoop/core/trunk/src/test/testshell/ExternalMapReduce.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/testshell/ExternalMapReduce.java?rev=695844&r1=695843&r2=695844&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/testshell/ExternalMapReduce.java (original)
+++ hadoop/core/trunk/src/test/testshell/ExternalMapReduce.java Tue Sep 16 05:47:39 2008
@@ -104,6 +104,16 @@
Path outDir = new Path(argv[1]);
Path input = new Path(argv[0]);
JobConf testConf = new JobConf(getConf(), ExternalMapReduce.class);
+
+ //try to load a class from libjar
+ try {
+ testConf.getClassByName("testjar.ClassWordCount");
+ } catch (ClassNotFoundException e) {
+ System.out.println("Could not find class from libjar");
+ return -1;
+ }
+
+
testConf.setJobName("external job");
FileInputFormat.setInputPaths(testConf, input);
FileOutputFormat.setOutputPath(testConf, outDir);