You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by tg...@apache.org on 2013/02/01 16:42:56 UTC
svn commit: r1441492 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclie...
Author: tgraves
Date: Fri Feb 1 15:42:55 2013
New Revision: 1441492
URL: http://svn.apache.org/viewvc?rev=1441492&view=rev
Log:
MAPREDUCE-3952. In MR2, when Total input paths to process == 1,CombinefileInputFormat.getSplits() returns 0 split. (Bhallamudi Venkata Siva Kamesh via tgraves)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1441492&r1=1441491&r2=1441492&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Feb 1 15:42:55 2013
@@ -27,6 +27,10 @@ Release 0.23.7 - UNRELEASED
MAPREDUCE-4458. Warn if java.library.path is used for AM or Task
(Robert Parker via jeagles)
+ MAPREDUCE-3952. In MR2, when Total input paths to process == 1,
+ CombinefileInputFormat.getSplits() returns 0 split. (Bhallamudi
+ Venkata Siva Kamesh via tgraves)
+
Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java?rev=1441492&r1=1441491&r2=1441492&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java Fri Feb 1 15:42:55 2013
@@ -498,6 +498,7 @@ public abstract class CombineFileInputFo
if (locations == null) {
blocks = new OneBlockInfo[0];
} else {
+
if (!isSplitable) {
// if the file is not splitable, just create the one block with
// full file length
@@ -515,7 +516,7 @@ public abstract class CombineFileInputFo
long left = locations[i].getLength();
long myOffset = locations[i].getOffset();
long myLength = 0;
- while (left > 0) {
+ do {
if (maxSize == 0) {
myLength = left;
} else {
@@ -537,7 +538,7 @@ public abstract class CombineFileInputFo
myOffset += myLength;
blocksList.add(oneblock);
- }
+ } while (left > 0);
}
blocks = blocksList.toArray(new OneBlockInfo[blocksList.size()]);
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java?rev=1441492&r1=1441491&r2=1441492&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java Fri Feb 1 15:42:55 2013
@@ -1147,6 +1147,35 @@ public class TestCombineFileInputFormat
}
}
+
+ /**
+ * Test when the input file's length is 0.
+ */
+ @Test
+ public void testForEmptyFile() throws Exception {
+ Configuration conf = new Configuration();
+ FileSystem fileSys = FileSystem.get(conf);
+ Path file = new Path("test" + "/file");
+ FSDataOutputStream out = fileSys.create(file, true,
+ conf.getInt("io.file.buffer.size", 4096), (short) 1, (long) BLOCKSIZE);
+ out.write(new byte[0]);
+ out.close();
+
+ // split it using a CombinedFile input format
+ DummyInputFormat inFormat = new DummyInputFormat();
+ Job job = Job.getInstance(conf);
+ FileInputFormat.setInputPaths(job, "test");
+ List<InputSplit> splits = inFormat.getSplits(job);
+ assertEquals(1, splits.size());
+ CombineFileSplit fileSplit = (CombineFileSplit) splits.get(0);
+ assertEquals(1, fileSplit.getNumPaths());
+ assertEquals(file.getName(), fileSplit.getPath(0).getName());
+ assertEquals(0, fileSplit.getOffset(0));
+ assertEquals(0, fileSplit.getLength(0));
+
+ fileSys.delete(file.getParent(), true);
+ }
+
static class TestFilter implements PathFilter {
private Path p;