You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2013/12/10 00:31:31 UTC
svn commit: r1549705 -
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
Author: jlowe
Date: Mon Dec 9 23:31:30 2013
New Revision: 1549705
URL: http://svn.apache.org/r1549705
Log:
MAPREDUCE-5656. bzip2 codec can drop records when reading data in splits. Contributed by Jason Lowe
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java?rev=1549705&r1=1549704&r2=1549705&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java Mon Dec 9 23:31:30 2013
@@ -175,6 +175,11 @@ public class LineReader implements Close
}
}
+ protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter)
+ throws IOException {
+ return in.read(buffer);
+ }
+
/**
* Read a line terminated by one of CR, LF, or CRLF.
*/
@@ -208,7 +213,7 @@ public class LineReader implements Close
if (prevCharCR) {
++bytesConsumed; //account for CR from previous read
}
- bufferLength = in.read(buffer);
+ bufferLength = fillBuffer(in, buffer, prevCharCR);
if (bufferLength <= 0) {
break; // EOF
}
@@ -296,7 +301,7 @@ public class LineReader implements Close
int startPosn = bufferPosn; // Start from previous end position
if (bufferPosn >= bufferLength) {
startPosn = bufferPosn = 0;
- bufferLength = in.read(buffer);
+ bufferLength = fillBuffer(in, buffer, ambiguousByteCount > 0);
if (bufferLength <= 0) {
str.append(recordDelimiterBytes, 0, ambiguousByteCount);
break; // EOF