You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/05/04 02:13:33 UTC
svn commit: r399500 - in /lucene/hadoop/trunk: ./ bin/
src/java/org/apache/hadoop/conf/ src/java/org/apache/hadoop/dfs/
src/java/org/apache/hadoop/io/ src/java/org/apache/hadoop/mapred/
src/java/org/apache/hadoop/util/
Author: cutting
Date: Wed May 3 17:13:31 2006
New Revision: 399500
URL: http://svn.apache.org/viewcvs?rev=399500&view=rev
Log:
Better fix for HADOOP-189. Tested in both distributed and standalone mode. Note that WritableFactories is not quite functioning correctly and there are a few hacks around this that should be removed when that's better understood.
Added:
lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java
- copied, changed from r399426, lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/bin/hadoop
lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ObjectWritable.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableName.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/CHANGES.txt?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed May 3 17:13:31 2006
@@ -167,6 +167,11 @@
'bin/hadoop jar build/hadoop-streaming.jar' to get details.
(Michel Tourn via cutting)
+44. HADOOP-189. Fix MapReduce in standalone configuration to
+ correctly handle job jar files that contain a lib directory with
+ nested jar files. (cutting)
+
+
Release 0.1.1 - 2006-04-08
1. Added CHANGES.txt, logging all significant changes to Hadoop. (cutting)
Modified: lucene/hadoop/trunk/bin/hadoop
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/bin/hadoop?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/bin/hadoop (original)
+++ lucene/hadoop/trunk/bin/hadoop Wed May 3 17:13:31 2006
@@ -135,14 +135,7 @@
elif [ "$COMMAND" = "job" ] ; then
CLASS=org.apache.hadoop.mapred.JobClient
elif [ "$COMMAND" = "jar" ] ; then
- JAR="$1"
- shift
- CLASS=`"$0" org.apache.hadoop.util.PrintJarMainClass "$JAR"`
- if [ $? != 0 ]; then
- echo "Error: Could not find main class in jar file $JAR"
- exit 1
- fi
- CLASSPATH=${CLASSPATH}:${JAR}
+ CLASS=org.apache.hadoop.util.RunJar
else
CLASS=$COMMAND
fi
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/conf/Configuration.java Wed May 3 17:13:31 2006
@@ -241,7 +241,7 @@
if (valueString == null)
return defaultValue;
try {
- return Class.forName(valueString);
+ return classLoader.loadClass(valueString);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Wed May 3 17:13:31 2006
@@ -49,6 +49,9 @@
Daemon leaseChecker;
private Configuration conf;
+ // required for unknown reason to make WritableFactories work distributed
+ static { new DFSFileInfo(); }
+
/**
* A map from name -> DFSOutputStream of files that are currently being
* written by this client.
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java Wed May 3 17:13:31 2006
@@ -65,6 +65,9 @@
//private static long numGigs = Configuration.get().getLong("dfs.datanode.maxgigs", 100);
//
+ // required for unknown reason to make WritableFactories work distributed
+ static { new BlockCommand(); }
+
/**
* Util method to build socket addr from string
*/
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ObjectWritable.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ObjectWritable.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ObjectWritable.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/ObjectWritable.java Wed May 3 17:13:31 2006
@@ -88,7 +88,8 @@
declaredClass = (Class)PRIMITIVE_NAMES.get(className);
if (declaredClass == null) {
try {
- declaredClass = Class.forName(className);
+ declaredClass =
+ Thread.currentThread().getContextClassLoader().loadClass(className);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e.toString());
}
@@ -170,7 +171,8 @@
Class declaredClass = (Class)PRIMITIVE_NAMES.get(className);
if (declaredClass == null) {
try {
- declaredClass = Class.forName(className);
+ declaredClass =
+ Thread.currentThread().getContextClassLoader().loadClass(className);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e.toString());
}
@@ -215,7 +217,8 @@
} else { // Writable
Class instanceClass = null;
try {
- instanceClass = Class.forName(UTF8.readString(in));
+ instanceClass = Thread.currentThread().getContextClassLoader()
+ .loadClass(UTF8.readString(in));
} catch (ClassNotFoundException e) {
throw new RuntimeException(e.toString());
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableName.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableName.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableName.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableName.java Wed May 3 17:13:31 2006
@@ -62,7 +62,7 @@
if (writableClass != null)
return writableClass;
try {
- return Class.forName(name);
+ return Thread.currentThread().getContextClassLoader().loadClass(name);
} catch (ClassNotFoundException e) {
throw new IOException(e.toString());
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java Wed May 3 17:13:31 2006
@@ -38,6 +38,9 @@
static long MAX_JOBPROFILE_AGE = 1000 * 2;
+ // required for unknown reason to make WritableFactories work distributed
+ static { new JobStatus(); new JobProfile(); }
+
/**
* A NetworkedJob is an implementation of RunningJob. It holds
* a JobProfile object to provide some info, and interacts with the
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Wed May 3 17:13:31 2006
@@ -49,6 +49,10 @@
idFormat.setMinimumIntegerDigits(4);
idFormat.setGroupingUsed(false);
}
+
+ // required for unknown reason to make WritableFactories work distributed
+ static { new TaskTrackerStatus(); }
+
private int nextJobId = 1;
public static final Logger LOG = LogFormatter.getLogger("org.apache.hadoop.mapred.JobTracker");
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskRunner.java Wed May 3 17:13:31 2006
@@ -18,6 +18,7 @@
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.fs.*;
+import org.apache.hadoop.util.*;
import java.io.*;
import java.util.jar.*;
@@ -78,7 +79,7 @@
String jar = conf.getJar();
if (jar != null) { // if jar exists, it into workDir
- unJar(new File(jar), workDir);
+ RunJar.unJar(new File(jar), workDir);
File[] libs = new File(workDir, "lib").listFiles();
if (libs != null) {
for (int i = 0; i < libs.length; i++) {
@@ -220,37 +221,6 @@
}
return text;
- }
-
- private void unJar(File jarFile, File toDir) throws IOException {
- JarFile jar = new JarFile(jarFile);
- try {
- Enumeration entries = jar.entries();
- while (entries.hasMoreElements()) {
- JarEntry entry = (JarEntry)entries.nextElement();
- if (!entry.isDirectory()) {
- InputStream in = jar.getInputStream(entry);
- try {
- File file = new File(toDir, entry.getName());
- file.getParentFile().mkdirs();
- OutputStream out = new FileOutputStream(file);
- try {
- byte[] buffer = new byte[8192];
- int i;
- while ((i = in.read(buffer)) != -1) {
- out.write(buffer, 0, i);
- }
- } finally {
- out.close();
- }
- } finally {
- in.close();
- }
- }
- }
- } finally {
- jar.close();
- }
}
/**
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=399500&r1=399499&r2=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Wed May 3 17:13:31 2006
@@ -38,6 +38,9 @@
static final int STALE_STATE = 1;
+ // required for unknown reason to make WritableFactories work distributed
+ static { new MapTask(); new ReduceTask(); new MapOutputLocation(); }
+
public static final Logger LOG =
LogFormatter.getLogger("org.apache.hadoop.mapred.TaskTracker");
Copied: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java (from r399426, lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java)
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java?p2=lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java&p1=lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java&r1=399426&r2=399500&rev=399500&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/RunJar.java Wed May 3 17:13:31 2006
@@ -107,14 +107,15 @@
unJar(file, workDir);
ArrayList classPath = new ArrayList();
+ classPath.add(new File(workDir+"/").toURL());
+ classPath.add(file.toURL());
+ classPath.add(new File(workDir, "classes/").toURL());
File[] libs = new File(workDir, "lib").listFiles();
if (libs != null) {
for (int i = 0; i < libs.length; i++) {
classPath.add(libs[i].toURL());
}
}
- classPath.add(new File(workDir, "classes/").toURL());
- classPath.add(new File(workDir+"/").toURL());
ClassLoader loader =
new URLClassLoader((URL[])classPath.toArray(new URL[0]));