You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by tg...@apache.org on 2013/02/01 16:50:44 UTC
svn commit: r1441494 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclie...
Author: tgraves
Date: Fri Feb 1 15:50:43 2013
New Revision: 1441494
URL: http://svn.apache.org/viewvc?rev=1441494&view=rev
Log:
MAPREDUCE-4470. Fix TestCombineFileInputFormat.testForEmptyFile (ikatsov via tgraves)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1441494&r1=1441493&r2=1441494&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Feb 1 15:50:43 2013
@@ -31,6 +31,9 @@ Release 0.23.7 - UNRELEASED
CombinefileInputFormat.getSplits() returns 0 split. (Bhallamudi
Venkata Siva Kamesh via tgraves)
+ MAPREDUCE-4470. Fix TestCombineFileInputFormat.testForEmptyFile (ikatsov
+ via tgraves)
+
Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java?rev=1441494&r1=1441493&r2=1441494&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java Fri Feb 1 15:50:43 2013
@@ -499,6 +499,10 @@ public abstract class CombineFileInputFo
blocks = new OneBlockInfo[0];
} else {
+ if(locations.length == 0) {
+ locations = new BlockLocation[] { new BlockLocation() };
+ }
+
if (!isSplitable) {
// if the file is not splitable, just create the one block with
// full file length
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java?rev=1441494&r1=1441493&r2=1441494&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java Fri Feb 1 15:50:43 2013
@@ -33,7 +33,9 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
@@ -136,6 +138,43 @@ public class TestFileInputFormat {
}
}
+ /**
+ * Test when the input file's length is 0.
+ */
+ @Test
+ public void testForEmptyFile() throws Exception {
+ Configuration conf = new Configuration();
+ FileSystem fileSys = FileSystem.get(conf);
+ Path file = new Path("test" + "/file");
+ FSDataOutputStream out = fileSys.create(file, true,
+ conf.getInt("io.file.buffer.size", 4096), (short) 1, (long) 1024);
+ out.write(new byte[0]);
+ out.close();
+
+ // split it using a File input format
+ DummyInputFormat inFormat = new DummyInputFormat();
+ Job job = Job.getInstance(conf);
+ FileInputFormat.setInputPaths(job, "test");
+ List<InputSplit> splits = inFormat.getSplits(job);
+ assertEquals(1, splits.size());
+ FileSplit fileSplit = (FileSplit) splits.get(0);
+ assertEquals(0, fileSplit.getLocations().length);
+ assertEquals(file.getName(), fileSplit.getPath().getName());
+ assertEquals(0, fileSplit.getStart());
+ assertEquals(0, fileSplit.getLength());
+
+ fileSys.delete(file.getParent(), true);
+ }
+
+ /** Dummy class to extend FileInputFormat*/
+ private class DummyInputFormat extends FileInputFormat<Text, Text> {
+ @Override
+ public RecordReader<Text,Text> createRecordReader(InputSplit split,
+ TaskAttemptContext context) throws IOException {
+ return null;
+ }
+ }
+
private class FileInputFormatForTest<K, V> extends FileInputFormat<K, V> {
long splitSize;