You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/05/17 21:04:02 UTC
svn commit: r407355 - in /lucene/hadoop/trunk: ./
src/java/org/apache/hadoop/mapred/
Author: cutting
Date: Wed May 17 12:04:01 2006
New Revision: 407355
URL: http://svn.apache.org/viewvc?rev=407355&view=rev
Log:
HADOOP-115. Permit reduce input and output types to differ. Contributed by Runping Qi.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=407355&r1=407354&r2=407355&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed May 17 12:04:01 2006
@@ -41,6 +41,10 @@
10. HADOOP-161. Add hashCode() method to DFS's Block.
(Milind Bhandarkar via cutting)
+11. HADOOP-115. Map output types may now be specified. These are also
+ used as reduce input types, thus permitting reduce input types to
+ differ from reduce output types. (Runping Qi via cutting)
+
Release 0.2.1 - 2006-05-12
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=407355&r1=407354&r2=407355&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Wed May 17 12:04:01 2006
@@ -269,14 +269,67 @@
public Class getInputValueClass() {
return getClass("mapred.input.value.class", UTF8.class, Writable.class);
}
+
public void setInputValueClass(Class theClass) {
setClass("mapred.input.value.class", theClass, Writable.class);
}
-
+
+ /**
+ * Get the key class for the map output data. If it is not set, use the
+ * (final) output ket class This allows the map output key class to be
+ * different than the final output key class
+ *
+ * @return map output key class
+ */
+ public Class getMapOutputKeyClass() {
+ Class retv = getClass("mapred.mapoutput.key.class", null,
+ WritableComparable.class);
+ if (retv == null) {
+ retv = getOutputKeyClass();
+ }
+ return retv;
+ }
+
+ /**
+ * Set the key class for the map output data. This allows the user to
+ * specify the map output key class to be different than the final output
+ * value class
+ */
+ public void setMapOutputKeyClass(Class theClass) {
+ setClass("mapred.mapoutput.key.class", theClass,
+ WritableComparable.class);
+ }
+
+ /**
+ * Get the value class for the map output data. If it is not set, use the
+ * (final) output value class This allows the map output value class to be
+ * different than the final output value class
+ *
+ * @return map output value class
+ */
+ public Class getMapOutputValueClass() {
+ Class retv = getClass("mapred.mapoutput.value.class", null,
+ Writable.class);
+ if (retv == null) {
+ retv = getOutputValueClass();
+ }
+ return retv;
+ }
+
+ /**
+ * Set the value class for the map output data. This allows the user to
+ * specify the map output value class to be different than the final output
+ * value class
+ */
+ public void setMapOutputValueClass(Class theClass) {
+ setClass("mapred.mapoutput.value.class", theClass, Writable.class);
+ }
+
public Class getOutputKeyClass() {
return getClass("mapred.output.key.class",
LongWritable.class, WritableComparable.class);
}
+
public void setOutputKeyClass(Class theClass) {
setClass("mapred.output.key.class", theClass, WritableComparable.class);
}
@@ -286,7 +339,7 @@
WritableComparator.class);
if (theClass != null)
return (WritableComparator)newInstance(theClass);
- return WritableComparator.get(getOutputKeyClass());
+ return WritableComparator.get(getMapOutputKeyClass());
}
public void setOutputKeyComparatorClass(Class theClass) {
@@ -300,6 +353,7 @@
public void setOutputValueClass(Class theClass) {
setClass("mapred.output.value.class", theClass, Writable.class);
}
+
public Class getMapperClass() {
return getClass("mapred.mapper.class", IdentityMapper.class, Mapper.class);
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java?rev=407355&r1=407354&r2=407355&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java Wed May 17 12:04:01 2006
@@ -38,8 +38,8 @@
final MapFile.Writer out =
new MapFile.Writer(fs, file.toString(),
- job.getOutputKeyClass(),
- job.getOutputValueClass(),
+ job.getMapOutputKeyClass(),
+ job.getMapOutputValueClass(),
job.getBoolean("mapred.output.compress", false));
return new RecordWriter() {
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java?rev=407355&r1=407354&r2=407355&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java Wed May 17 12:04:01 2006
@@ -77,8 +77,8 @@
outs[i] =
new SequenceFile.Writer(FileSystem.getNamed("local", job),
this.mapOutputFile.getOutputFile(getTaskId(), i),
- job.getOutputKeyClass(),
- job.getOutputValueClass());
+ job.getMapOutputKeyClass(),
+ job.getMapOutputValueClass());
}
final Partitioner partitioner =
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java?rev=407355&r1=407354&r2=407355&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java Wed May 17 12:04:01 2006
@@ -173,8 +173,8 @@
public void run(JobConf job, final TaskUmbilicalProtocol umbilical)
throws IOException {
- Class keyClass = job.getOutputKeyClass();
- Class valueClass = job.getOutputValueClass();
+ Class keyClass = job.getMapOutputKeyClass();
+ Class valueClass = job.getMapOutputValueClass();
Reducer reducer = (Reducer)job.newInstance(job.getReducerClass());
reducer.configure(job);
FileSystem lfs = FileSystem.getNamed("local", job);