You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ac...@apache.org on 2008/10/23 05:59:21 UTC
svn commit: r707262 - in /hadoop/core/trunk: CHANGES.txt
src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
src/test/org/apache/hadoop/mapred/TestJavaSerialization.java
Author: acmurthy
Date: Wed Oct 22 20:59:20 2008
New Revision: 707262
URL: http://svn.apache.org/viewvc?rev=707262&view=rev
Log:
HADOOP-4466. Ensure that SequenceFileOutputFormat isn't tied to Writables and can be used with other Serialization frameworks. Contributed by Chris Wensel.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=707262&r1=707261&r2=707262&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Oct 22 20:59:20 2008
@@ -1018,6 +1018,10 @@
HADOOP-4387. TestHDFSFileSystemContract fails on windows nightly builds.
(Raghu Angadi)
+ HADOOP-4466. Ensure that SequenceFileOutputFormat isn't tied to Writables
+ and can be used with other Serialization frameworks. (Chris Wensel via
+ acmurthy)
+
Release 0.18.2 - Unreleased
BUG FIXES
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java?rev=707262&r1=707261&r2=707262&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SequenceFileOutputFormat.java Wed Oct 22 20:59:20 2008
@@ -26,8 +26,6 @@
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
@@ -58,8 +56,8 @@
}
final SequenceFile.Writer out =
SequenceFile.createWriter(fs, job, file,
- job.getOutputKeyClass().asSubclass(WritableComparable.class),
- job.getOutputValueClass().asSubclass(Writable.class),
+ job.getOutputKeyClass(),
+ job.getOutputValueClass(),
compressionType,
codec,
progress);
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java?rev=707262&r1=707261&r2=707262&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJavaSerialization.java Wed Oct 22 20:59:20 2008
@@ -106,4 +106,46 @@
reader.close();
}
+ /**
+ * HADOOP-4466:
+ * This test verifies the JavSerialization impl can write to SequenceFiles. by virtue other
+ * SequenceFileOutputFormat is not coupled to Writable types, if so, the job will fail.
+ *
+ */
+ public void testWriteToSequencefile() throws Exception {
+ OutputStream os = getFileSystem().create(new Path(getInputDir(),
+ "text.txt"));
+ Writer wr = new OutputStreamWriter(os);
+ wr.write("b a\n");
+ wr.close();
+
+ JobConf conf = createJobConf();
+ conf.setJobName("JavaSerialization");
+
+ conf.set("io.serializations",
+ "org.apache.hadoop.io.serializer.JavaSerialization," +
+ "org.apache.hadoop.io.serializer.WritableSerialization");
+
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(SequenceFileOutputFormat.class); // test we can write to sequence files
+
+ conf.setOutputKeyClass(String.class);
+ conf.setOutputValueClass(Long.class);
+ conf.setOutputKeyComparatorClass(JavaSerializationComparator.class);
+
+ conf.setMapperClass(WordCountMapper.class);
+ conf.setReducerClass(SumReducer.class);
+
+ FileInputFormat.setInputPaths(conf, getInputDir());
+
+ FileOutputFormat.setOutputPath(conf, getOutputDir());
+
+ JobClient.runJob(conf);
+
+ Path[] outputFiles = FileUtil.stat2Paths(
+ getFileSystem().listStatus(getOutputDir(),
+ new OutputLogFilter()));
+ assertEquals(1, outputFiles.length);
+}
+
}