You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2016/05/15 14:50:45 UTC
[05/31] hadoop git commit: MAPREDUCE-6558. multibyte delimiters with
compressed input files generate duplicate records. Contributed by Wilfred
Spiegelenburg
MAPREDUCE-6558. multibyte delimiters with compressed input files generate duplicate records. Contributed by Wilfred Spiegelenburg
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9227dfc2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9227dfc2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9227dfc2
Branch: refs/heads/HADOOP-12930
Commit: 9227dfc25f373a99cb66ad7d6bacef8dcf336f77
Parents: dd3a8be
Author: Jason Lowe <jl...@apache.org>
Authored: Fri May 13 14:33:48 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri May 13 14:33:48 2016 +0000
----------------------------------------------------------------------
.../lib/input/CompressedSplitLineReader.java | 5 ++++
.../hadoop/mapred/TestLineRecordReader.java | 29 +++++++++++++++++++
.../lib/input/TestLineRecordReader.java | 29 +++++++++++++++++++
.../compressedMultibyteDelimiter.txt.bz2 | Bin 0 -> 1096 bytes
4 files changed, 63 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9227dfc2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
index ef51f5c..9d0e949 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
@@ -165,4 +165,9 @@ public class CompressedSplitLineReader extends SplitLineReader {
public boolean needAdditionalRecordAfterSplit() {
return !finished && needAdditionalRecord;
}
+
+ @Override
+ protected void unsetNeedAdditionalRecordAfterSplit() {
+ needAdditionalRecord = false;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9227dfc2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
index f50e1ef..844250b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
@@ -654,4 +654,33 @@ public class TestLineRecordReader {
assertFalse(reader.next(key, value));
assertEquals(12, reader.getPos());
}
+
+ @Test
+ public void testBzipWithMultibyteDelimiter() throws IOException {
+ String testFileName = "compressedMultibyteDelimiter.txt.bz2";
+ // firstSplitLength < (headers + blockMarker) will pass always since no
+ // records will be read (in the test file that is byte 0..9)
+ // firstSplitlength > (compressed file length - one compressed block
+ // size + 1) will also always pass since the second split will be empty
+ // (833 bytes is the last block start in the used data file)
+ int firstSplitLength = 100;
+ URL testFileUrl = getClass().getClassLoader().getResource(testFileName);
+ assertNotNull("Cannot find " + testFileName, testFileUrl);
+ File testFile = new File(testFileUrl.getFile());
+ long testFileSize = testFile.length();
+ Path testFilePath = new Path(testFile.getAbsolutePath());
+ assertTrue("Split size is smaller than header length",
+ firstSplitLength > 9);
+ assertTrue("Split size is larger than compressed file size " +
+ testFilePath, testFileSize > firstSplitLength);
+
+ Configuration conf = new Configuration();
+ conf.setInt(org.apache.hadoop.mapreduce.lib.input.
+ LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);
+
+ String delimiter = "<E-LINE>\r\r\n";
+ conf.set("textinputformat.record.delimiter", delimiter);
+ testSplitRecordsForFile(conf, firstSplitLength, testFileSize,
+ testFilePath);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9227dfc2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
index 6819af7..716c4a6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
@@ -617,4 +617,33 @@ public class TestLineRecordReader {
// Key should be 12 right after "123456789\r\r\n"
assertEquals(12, key.get());
}
+
+ @Test
+ public void testBzipWithMultibyteDelimiter() throws IOException {
+ String testFileName = "compressedMultibyteDelimiter.txt.bz2";
+ // firstSplitLength < (headers + blockMarker) will pass always since no
+ // records will be read (in the test file that is byte 0..9)
+ // firstSplitlength > (compressed file length - one compressed block
+ // size + 1) will also always pass since the second split will be empty
+ // (833 bytes is the last block start in the used data file)
+ int firstSplitLength = 100;
+ URL testFileUrl = getClass().getClassLoader().getResource(testFileName);
+ assertNotNull("Cannot find " + testFileName, testFileUrl);
+ File testFile = new File(testFileUrl.getFile());
+ long testFileSize = testFile.length();
+ Path testFilePath = new Path(testFile.getAbsolutePath());
+ assertTrue("Split size is smaller than header length",
+ firstSplitLength > 9);
+ assertTrue("Split size is larger than compressed file size " +
+ testFilePath, testFileSize > firstSplitLength);
+
+ Configuration conf = new Configuration();
+ conf.setInt(org.apache.hadoop.mapreduce.lib.input.
+ LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);
+
+ String delimiter = "<E-LINE>\r\r\n";
+ conf.set("textinputformat.record.delimiter", delimiter);
+ testSplitRecordsForFile(conf, firstSplitLength, testFileSize,
+ testFilePath);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9227dfc2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/compressedMultibyteDelimiter.txt.bz2
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/compressedMultibyteDelimiter.txt.bz2 b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/compressedMultibyteDelimiter.txt.bz2
new file mode 100644
index 0000000..f8e178f
Binary files /dev/null and b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/compressedMultibyteDelimiter.txt.bz2 differ
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org