You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2008/09/26 00:25:27 UTC
svn commit: r699113 - in /hadoop/core/trunk: ./
src/contrib/streaming/src/java/org/apache/hadoop/streaming/
src/contrib/streaming/src/test/org/apache/hadoop/streaming/
src/core/org/apache/hadoop/util/ src/mapred/org/apache/hadoop/mapred/
src/mapred/org...
Author: omalley
Date: Thu Sep 25 15:25:24 2008
New Revision: 699113
URL: http://svn.apache.org/viewvc?rev=699113&view=rev
Log:
HADOOP-4269. Fix the deprecation of LineReader by extending the new class
into the old name and deprecating it. Also update the tests to test the
new class. (cdouglas via omalley)
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamKeyValUtil.java
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java
hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamedMerge.java
hadoop/core/trunk/src/core/org/apache/hadoop/util/LineReader.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/NLineInputFormat.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=699113&r1=699112&r2=699113&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Sep 25 15:25:24 2008
@@ -748,6 +748,10 @@
HADOOP-4247. Move InputSampler into org.apache.hadoop.mapred.lib, so that
examples.jar doesn't depend on tools.jar. (omalley)
+ HADOOP-4269. Fix the deprecation of LineReader by extending the new class
+ into the old name and deprecating it. Also update the tests to test the
+ new class. (cdouglas via omalley)
+
Release 0.18.2 - Unreleased
BUG FIXES
Modified: hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java?rev=699113&r1=699112&r2=699113&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java (original)
+++ hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java Thu Sep 25 15:25:24 2008
@@ -34,7 +34,7 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.UTF8ByteArrayUtils;
Modified: hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamKeyValUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamKeyValUtil.java?rev=699113&r1=699112&r2=699113&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamKeyValUtil.java (original)
+++ hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamKeyValUtil.java Thu Sep 25 15:25:24 2008
@@ -21,7 +21,7 @@
import java.io.IOException;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
public class StreamKeyValUtil {
Modified: hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java?rev=699113&r1=699112&r2=699113&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java (original)
+++ hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java Thu Sep 25 15:25:24 2008
@@ -21,8 +21,7 @@
import java.io.IOException;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.LineRecordReader;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
/**
* General utils for byte array containing UTF-8 encoded strings
@@ -205,7 +204,7 @@
* @param out Text to read into
* @return number of bytes read
* @deprecated use
- * {@link StreamKeyValUtil#readLine(LineRecordReader.LineReader, Text)}
+ * {@link StreamKeyValUtil#readLine(LineReader, Text)}
* @throws IOException
*/
@Deprecated
Modified: hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamedMerge.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamedMerge.java?rev=699113&r1=699112&r2=699113&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamedMerge.java (original)
+++ hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamedMerge.java Thu Sep 25 15:25:24 2008
@@ -36,7 +36,7 @@
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
import org.apache.hadoop.util.ToolRunner;
/**
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/LineReader.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/LineReader.java?rev=699113&r1=699112&r2=699113&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/LineReader.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/LineReader.java Thu Sep 25 15:25:24 2008
@@ -38,12 +38,23 @@
private int bufferPosn = 0;
/**
+ * Create a line reader that reads from the given stream using the
+ * default buffer-size (64k).
+ * @param in The input stream
+ * @throws IOException
+ */
+ public LineReader(InputStream in) {
+ this(in, DEFAULT_BUFFER_SIZE);
+ }
+
+ /**
* Create a line reader that reads from the given stream using the
* given buffer-size.
- * @param in
+ * @param in The input stream
+ * @param bufferSize Size of the read buffer
* @throws IOException
*/
- LineReader(InputStream in, int bufferSize) {
+ public LineReader(InputStream in, int bufferSize) {
this.in = in;
this.bufferSize = bufferSize;
this.buffer = new byte[this.bufferSize];
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java?rev=699113&r1=699112&r2=699113&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java Thu Sep 25 15:25:24 2008
@@ -52,145 +52,16 @@
* @deprecated Use {@link org.apache.hadoop.util.LineReader} instead.
*/
@Deprecated
- public static class LineReader {
- private static final int DEFAULT_BUFFER_SIZE = 64 * 1024;
- private int bufferSize = DEFAULT_BUFFER_SIZE;
- private InputStream in;
- private byte[] buffer;
- // the number of bytes of real data in the buffer
- private int bufferLength = 0;
- // the current position in the buffer
- private int bufferPosn = 0;
-
- /**
- * Create a line reader that reads from the given stream using the
- * given buffer-size.
- * @param in
- * @throws IOException
- */
+ public static class LineReader extends org.apache.hadoop.util.LineReader {
+ LineReader(InputStream in) {
+ super(in);
+ }
LineReader(InputStream in, int bufferSize) {
- this.in = in;
- this.bufferSize = bufferSize;
- this.buffer = new byte[this.bufferSize];
+ super(in, bufferSize);
}
-
- /**
- * Create a line reader that reads from the given stream using the
- * <code>io.file.buffer.size</code> specified in the given
- * <code>Configuration</code>.
- * @param in input stream
- * @param conf configuration
- * @throws IOException
- */
public LineReader(InputStream in, Configuration conf) throws IOException {
- this(in, conf.getInt("io.file.buffer.size", DEFAULT_BUFFER_SIZE));
+ super(in, conf);
}
-
- /**
- * Fill the buffer with more data.
- * @return was there more data?
- * @throws IOException
- */
- boolean backfill() throws IOException {
- bufferPosn = 0;
- bufferLength = in.read(buffer);
- return bufferLength > 0;
- }
-
- /**
- * Close the underlying stream.
- * @throws IOException
- */
- public void close() throws IOException {
- in.close();
- }
-
- /**
- * Read from the InputStream into the given Text.
- * @param str the object to store the given line
- * @param maxLineLength the maximum number of bytes to store into str.
- * @param maxBytesToConsume the maximum number of bytes to consume in this call.
- * @return the number of bytes read including the newline
- * @throws IOException if the underlying stream throws
- */
- public int readLine(Text str, int maxLineLength,
- int maxBytesToConsume) throws IOException {
- str.clear();
- boolean hadFinalNewline = false;
- boolean hadFinalReturn = false;
- boolean hitEndOfFile = false;
- int startPosn = bufferPosn;
- long bytesConsumed = 0;
- outerLoop: while (true) {
- if (bufferPosn >= bufferLength) {
- if (!backfill()) {
- hitEndOfFile = true;
- break;
- }
- }
- startPosn = bufferPosn;
- for(; bufferPosn < bufferLength; ++bufferPosn) {
- switch (buffer[bufferPosn]) {
- case '\n':
- hadFinalNewline = true;
- bufferPosn += 1;
- break outerLoop;
- case '\r':
- if (hadFinalReturn) {
- // leave this \r in the stream, so we'll get it next time
- break outerLoop;
- }
- hadFinalReturn = true;
- break;
- default:
- if (hadFinalReturn) {
- break outerLoop;
- }
- }
- }
- bytesConsumed += bufferPosn - startPosn;
- int length = bufferPosn - startPosn - (hadFinalReturn ? 1 : 0);
- length = (int)Math.min(length, maxLineLength - str.getLength());
- if (length >= 0) {
- str.append(buffer, startPosn, length);
- }
- if (bytesConsumed >= maxBytesToConsume) {
- return (int)Math.min(bytesConsumed, (long)Integer.MAX_VALUE);
- }
- }
- int newlineLength = (hadFinalNewline ? 1 : 0) + (hadFinalReturn ? 1 : 0);
- if (!hitEndOfFile) {
- bytesConsumed += bufferPosn - startPosn;
- int length = bufferPosn - startPosn - newlineLength;
- length = (int)Math.min(length, maxLineLength - str.getLength());
- if (length > 0) {
- str.append(buffer, startPosn, length);
- }
- }
- return (int)Math.min(bytesConsumed, (long)Integer.MAX_VALUE);
- }
-
- /**
- * Read from the InputStream into the given Text.
- * @param str the object to store the given line
- * @param maxLineLength the maximum number of bytes to store into str.
- * @return the number of bytes read including the newline
- * @throws IOException if the underlying stream throws
- */
- public int readLine(Text str, int maxLineLength) throws IOException {
- return readLine(str, maxLineLength, Integer.MAX_VALUE);
- }
-
- /**
- * Read from the InputStream into the given Text.
- * @param str the object to store the given line
- * @return the number of bytes read including the newline
- * @throws IOException if the underlying stream throws
- */
- public int readLine(Text str) throws IOException {
- return readLine(str, Integer.MAX_VALUE, Integer.MAX_VALUE);
- }
-
}
public LineRecordReader(Configuration job,
@@ -228,7 +99,7 @@
public LineRecordReader(InputStream in, long offset, long endOffset,
int maxLineLength) {
this.maxLineLength = maxLineLength;
- this.in = new LineReader(in, LineReader.DEFAULT_BUFFER_SIZE);
+ this.in = new LineReader(in);
this.start = offset;
this.pos = offset;
this.end = endOffset;
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/NLineInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/NLineInputFormat.java?rev=699113&r1=699112&r2=699113&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/NLineInputFormat.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/NLineInputFormat.java Thu Sep 25 15:25:24 2008
@@ -35,7 +35,7 @@
import org.apache.hadoop.mapred.LineRecordReader;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
/**
* NLineInputFormat which splits N lines of input as one split.
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java?rev=699113&r1=699112&r2=699113&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java Thu Sep 25 15:25:24 2008
@@ -26,7 +26,7 @@
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.io.compress.*;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
import org.apache.hadoop.util.ReflectionUtils;
public class TestKeyValueTextInputFormat extends TestCase {
@@ -130,7 +130,7 @@
}
}
private LineReader makeStream(String str) throws IOException {
- return new LineRecordReader.LineReader(new ByteArrayInputStream
+ return new LineReader(new ByteArrayInputStream
(str.getBytes("UTF-8")),
defaultConf);
}
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java?rev=699113&r1=699112&r2=699113&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java Thu Sep 25 15:25:24 2008
@@ -26,7 +26,7 @@
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.io.compress.*;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
import org.apache.hadoop.util.ReflectionUtils;
public class TestTextInputFormat extends TestCase {
@@ -128,7 +128,7 @@
}
private static LineReader makeStream(String str) throws IOException {
- return new LineRecordReader.LineReader(new ByteArrayInputStream
+ return new LineReader(new ByteArrayInputStream
(str.getBytes("UTF-8")),
defaultConf);
}