You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ac...@apache.org on 2008/10/23 06:01:33 UTC
svn commit: r707263 - in /hadoop/core/branches/branch-0.19: ./ CHANGES.txt
src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
src/test/org/apache/hadoop/mapred/TestJavaSerialization.java
Author: acmurthy
Date: Wed Oct 22 21:01:32 2008
New Revision: 707263
URL: http://svn.apache.org/viewvc?rev=707263&view=rev
Log:
Merge -r 707261:707262 from trunk to branch-0.19 to fix HADOOP-4466.
Modified:
hadoop/core/branches/branch-0.19/ (props changed)
hadoop/core/branches/branch-0.19/CHANGES.txt
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java
Propchange: hadoop/core/branches/branch-0.19/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 22 21:01:32 2008
@@ -1 +1 @@
-/hadoop/core/trunk:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258
+/hadoop/core/trunk:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262
Modified: hadoop/core/branches/branch-0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/CHANGES.txt?rev=707263&r1=707262&r2=707263&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.19/CHANGES.txt Wed Oct 22 21:01:32 2008
@@ -956,6 +956,10 @@
HADOOP-4387. TestHDFSFileSystemContract fails on windows nightly builds.
(Raghu Angadi)
+ HADOOP-4466. Ensure that SequenceFileOutputFormat isn't tied to Writables
+ and can be used with other Serialization frameworks. (Chris Wensel via
+ acmurthy)
+
Release 0.18.2 - Unreleased
BUG FIXES
Modified: hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java?rev=707263&r1=707262&r2=707263&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java (original)
+++ hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java Wed Oct 22 21:01:32 2008
@@ -26,8 +26,6 @@
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
@@ -58,8 +56,8 @@
}
final SequenceFile.Writer out =
SequenceFile.createWriter(fs, job, file,
- job.getOutputKeyClass().asSubclass(WritableComparable.class),
- job.getOutputValueClass().asSubclass(Writable.class),
+ job.getOutputKeyClass(),
+ job.getOutputValueClass(),
compressionType,
codec,
progress);
Modified: hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java?rev=707263&r1=707262&r2=707263&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java (original)
+++ hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java Wed Oct 22 21:01:32 2008
@@ -106,4 +106,46 @@
reader.close();
}
+ /**
+ * HADOOP-4466:
+ * This test verifies the JavSerialization impl can write to SequenceFiles. by virtue other
+ * SequenceFileOutputFormat is not coupled to Writable types, if so, the job will fail.
+ *
+ */
+ public void testWriteToSequencefile() throws Exception {
+ OutputStream os = getFileSystem().create(new Path(getInputDir(),
+ "text.txt"));
+ Writer wr = new OutputStreamWriter(os);
+ wr.write("b a\n");
+ wr.close();
+
+ JobConf conf = createJobConf();
+ conf.setJobName("JavaSerialization");
+
+ conf.set("io.serializations",
+ "org.apache.hadoop.io.serializer.JavaSerialization," +
+ "org.apache.hadoop.io.serializer.WritableSerialization");
+
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(SequenceFileOutputFormat.class); // test we can write to sequence files
+
+ conf.setOutputKeyClass(String.class);
+ conf.setOutputValueClass(Long.class);
+ conf.setOutputKeyComparatorClass(JavaSerializationComparator.class);
+
+ conf.setMapperClass(WordCountMapper.class);
+ conf.setReducerClass(SumReducer.class);
+
+ FileInputFormat.setInputPaths(conf, getInputDir());
+
+ FileOutputFormat.setOutputPath(conf, getOutputDir());
+
+ JobClient.runJob(conf);
+
+ Path[] outputFiles = FileUtil.stat2Paths(
+ getFileSystem().listStatus(getOutputDir(),
+ new OutputLogFilter()));
+ assertEquals(1, outputFiles.length);
+}
+
}