You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/08/11 20:04:59 UTC
svn commit: r430838 - in /lucene/hadoop/trunk: ./
src/contrib/smallJobsBenchmark/ src/contrib/smallJobsBenchmark/bin/
src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/
Author: cutting
Date: Fri Aug 11 11:04:58 2006
New Revision: 430838
URL: http://svn.apache.org/viewvc?rev=430838&view=rev
Log:
HADOOP-434. Change smallJobsBenchmark to user standard Hadoop scripts. Contributed by Sanjay Dahiya.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/Readme.txt
lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/bin/run.sh
lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/build.xml
lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=430838&r1=430837&r2=430838&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Aug 11 11:04:58 2006
@@ -19,6 +19,9 @@
4. HADOOP-324. Fix datanode to not exit when a disk is full, but
rather simply to fail writes. (Wendy Chien via cutting)
+ 5. HADOOP-434. Change smallJobsBenchmark to use standard Hadoop
+ scripts. (Sanjay Dahiya via cutting)
+
Release 0.5.0 - 2006-08-04
Modified: lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/Readme.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/Readme.txt?rev=430838&r1=430837&r2=430838&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/Readme.txt (original)
+++ lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/Readme.txt Fri Aug 11 11:04:58 2006
@@ -6,6 +6,9 @@
$ ant deploy
Running the benchmark
+$ $HADOOP_HOME/bin/hadoop jar <path to MRBenchmark.jar> smallJobsBenchmark -inputLines <numLines> -maps <numMaps> -reduces <numReduces> -jar <path to MRBenchmark.jar>
+
+For running the benchmark multiple times with different input sizes, maps, reduces following helper scripts can be used.
$ cd build/contrib/smallJobsBenchmark
$ bin/run.sh
Modified: lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/bin/run.sh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/bin/run.sh?rev=430838&r1=430837&r2=430838&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/bin/run.sh (original)
+++ lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/bin/run.sh Fri Aug 11 11:04:58 2006
@@ -12,8 +12,6 @@
exit 1;
fi
-export CLASSPATH=$HADOOP_HOME/conf:$HADOOP_HOME/build/classes:$HADOOP_HOME/build:$HADOOP_HOME/build/test/classes:$HADOOP_HOME/hadoop-*.jar:$HADOOP_HOME/lib/commons-cli-2.0-SNAPSHOT.jar:$HADOOP_HOME/lib/commons-logging-1.0.4.jar:$HADOOP_HOME/lib/commons-logging-api-1.0.4.jar:$HADOOP_HOME/lib/jetty-5.1.4.jar:$HADOOP_HOME/lib/junit-3.8.1.jar:$HADOOP_HOME/lib/log4j-1.2.13.jar:$HADOOP_HOME/lib/lucene-core-1.9.1.jar:$HADOOP_HOME/lib/servlet-api.jar:$HADOOP_HOME/lib/jetty-ext/ant.jar:$HADOOP_HOME/lib/jetty-ext/commons-el.jar:$HADOOP_HOME/lib/jetty-ext/jasper-compiler.jar:$HADOOP_HOME/lib/jetty-ext/jasper-runtime.jar:$HADOOP_HOME/lib/jetty-ext/jsp-api.jar
-
mkdir -p logs;
export TIMES=2
@@ -26,7 +24,7 @@
do
for reduces in 1 18
do
-$JAVA_HOME/bin/java -classpath $CLASSPATH:./classes org.apache.hadoop.benchmarks.mapred.MultiJobRunner -inputLines ${dataLines} -output /hadoop/mapred/MROutput -jar MRBenchmark.jar -times ${TIMES} -workDir /hadoop/mapred/work -maps ${maps} -reduces ${reduces} -inputType ascending -ignoreOutput 2>&1 | tee logs/benchmark_${dataLines}_${maps}_${reduces}.log
+$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/../contrib/smallJobsBenchmark/MRBenchmark.jar smallJobsBenchmark -inputLines ${dataLines} -output /hadoop/mapred/MROutput -jar $HADOOP_HOME/../contrib/smallJobsBenchmark/MRBenchmark.jar -times ${TIMES} -workDir /hadoop/mapred/work -maps ${maps} -reduces ${reduces} -inputType ascending -ignoreOutput 2>&1 | tee logs/benchmark_${dataLines}_${maps}_${reduces}.log
done
done
Modified: lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/build.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/build.xml?rev=430838&r1=430837&r2=430838&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/build.xml (original)
+++ lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/build.xml Fri Aug 11 11:04:58 2006
@@ -1,6 +1,6 @@
<?xml version="1.0"?>
-<project name="smallJobsBenchmark" default="jar">
+<project name="smallJobsBenchmark" default="deploy">
<import file="../build-contrib.xml"/>
@@ -12,6 +12,10 @@
<jar
jarfile="${build.dir}/${benchmarkJarName}"
basedir="${build.classes}">
+ <manifest>
+ <attribute name="Main-Class"
+ value="org/apache/hadoop/benchmarks/mapred/BenchmarkRunner"/>
+ </manifest>
</jar>
</target>
Modified: lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java?rev=430838&r1=430837&r2=430838&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java (original)
+++ lucene/hadoop/trunk/src/contrib/smallJobsBenchmark/src/java/org/apache/hadoop/benchmarks/mapred/MultiJobRunner.java Fri Aug 11 11:04:58 2006
@@ -26,10 +26,10 @@
*/
public class MultiJobRunner {
- private String jarFile = "MRBenchmark.jar" ;
+ private String jarFile = null ; // "MRBenchmark.jar" ;
private String input ;
private String output ;
- private int numJobs = 2000 ; // default value
+ private int numJobs = 1 ; // default value
private static final Log LOG = LogFactory.getLog(MultiJobRunner.class);
private int numMaps = 2;
private int numReduces = 1;
@@ -130,7 +130,9 @@
job.setOutputPath(new Path(output));
- job.setJar(jarFile);
+ if( null != jarFile ){
+ job.setJar(jarFile);
+ }
job.setMapperClass(BenchmarkMapper.class);
job.setReducerClass(BenchmarkReducer.class);
@@ -338,8 +340,8 @@
}
String output = "";
- String jarFile = "MRBenchmark.jar" ;
- int numJobs = 0 ;
+ String jarFile = null; //"MRBenchmark.jar" ;
+ int numJobs = 1 ;
int numMaps = 2;
int numReduces = 1 ;
int dataLines = 1 ;