You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2011/12/01 01:30:39 UTC
svn commit: r1208909 - in /hadoop/common/branches/branch-1: CHANGES.txt
src/core/org/apache/hadoop/io/SequenceFile.java
src/test/org/apache/hadoop/io/TestSequenceFile.java
Author: eli
Date: Thu Dec 1 00:30:38 2011
New Revision: 1208909
URL: http://svn.apache.org/viewvc?rev=1208909&view=rev
Log:
HADOOP-7870. fix SequenceFile#createWriter with boolean createParent arg to respect createParent. Contributed by Jon Hsieh
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/SequenceFile.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/io/TestSequenceFile.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1208909&r1=1208908&r2=1208909&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Thu Dec 1 00:30:38 2011
@@ -52,6 +52,9 @@ Release 1.1.0 - unreleased
MAPREDUCE-2376. test-task-controller fails if run as a userid < 1000.
(todd via eli)
+ HADOOP-7870. fix SequenceFile#createWriter with boolean
+ createParent arg to respect createParent. (Jon Hsieh via eli)
+
IMPROVEMENTS
MAPREDUCE-3008. [Gridmix] Improve cumulative CPU usage emulation for
Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/SequenceFile.java?rev=1208909&r1=1208908&r2=1208909&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/SequenceFile.java (original)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/SequenceFile.java Thu Dec 1 00:30:38 2011
@@ -437,19 +437,24 @@ public class SequenceFile {
"GzipCodec without native-hadoop code!");
}
+
+ FSDataOutputStream fsos;
+ if (createParent) {
+ fsos = fs.create(name, true, bufferSize, replication, blockSize);
+ } else {
+ fsos = fs.createNonRecursive(name, true, bufferSize, replication,
+ blockSize, null);
+ }
+
switch (compressionType) {
case NONE:
- return new Writer(conf,
- fs.createNonRecursive(name, true, bufferSize, replication, blockSize, null),
- keyClass, valClass, metadata).ownStream();
+ return new Writer(conf, fsos, keyClass, valClass, metadata).ownStream();
case RECORD:
- return new RecordCompressWriter(conf,
- fs.createNonRecursive(name, true, bufferSize, replication, blockSize, null),
- keyClass, valClass, codec, metadata).ownStream();
+ return new RecordCompressWriter(conf, fsos, keyClass, valClass, codec,
+ metadata).ownStream();
case BLOCK:
- return new BlockCompressWriter(conf,
- fs.createNonRecursive(name, true, bufferSize, replication, blockSize, null),
- keyClass, valClass, codec, metadata).ownStream();
+ return new BlockCompressWriter(conf, fsos, keyClass, valClass, codec,
+ metadata).ownStream();
default:
return null;
}
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/io/TestSequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/io/TestSequenceFile.java?rev=1208909&r1=1208908&r2=1208909&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/io/TestSequenceFile.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/io/TestSequenceFile.java Thu Dec 1 00:30:38 2011
@@ -26,6 +26,7 @@ import org.apache.commons.logging.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.SequenceFile.Metadata;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.util.ReflectionUtils;
@@ -440,6 +441,31 @@ public class TestSequenceFile extends Te
assertFalse(reader2.next(text));
}
+
+ public void testRecursiveSeqFileCreate() throws IOException {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ Path name = new Path(new Path(System.getProperty("test.build.data","."),
+ "recursiveCreateDir") , "file");
+ boolean createParent = false;
+
+ try {
+ SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
+ RandomDatum.class, 512, (short) 1, 4096, createParent,
+ CompressionType.NONE, null, new Metadata());
+ fail("Expected an IOException due to missing parent");
+ } catch (IOException ioe) {
+ // Expected
+ }
+
+ createParent = true;
+ SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
+ RandomDatum.class, 512, (short) 1, 4096, createParent,
+ CompressionType.NONE, null, new Metadata());
+
+ // should succeed, fails if exception thrown
+ }
+
/** For debugging and testing. */
public static void main(String[] args) throws Exception {
int count = 1024 * 1024;