You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2013/10/21 19:11:35 UTC
svn commit: r1534279 - in /hadoop/common/branches/HDFS-2832: ./
hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/
hadoop-tools/hadoop-streaming/src/test/...
Author: arp
Date: Mon Oct 21 17:11:34 2013
New Revision: 1534279
URL: http://svn.apache.org/r1534279
Log:
Merging r1533208 through r1534278 from trunk to branch HDFS-2832
Added:
hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/KeyOnlyTextInputWriter.java
- copied unchanged from r1534278, hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/KeyOnlyTextInputWriter.java
hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/KeyOnlyTextOutputReader.java
- copied unchanged from r1534278, hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/KeyOnlyTextOutputReader.java
hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputOnlyKeys.java
- copied unchanged from r1534278, hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputOnlyKeys.java
hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/io/
- copied from r1534278, hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/io/
Modified:
hadoop/common/branches/HDFS-2832/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/IdentifierResolver.java
hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/TextInputWriter.java
Propchange: hadoop/common/branches/HDFS-2832/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk:r1533208-1534278
Modified: hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/IdentifierResolver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/IdentifierResolver.java?rev=1534279&r1=1534278&r2=1534279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/IdentifierResolver.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/IdentifierResolver.java Mon Oct 21 17:11:34 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.streaming.io;
import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.typedbytes.TypedBytesWritable;
@@ -34,6 +35,7 @@ public class IdentifierResolver {
public static final String TEXT_ID = "text";
public static final String RAW_BYTES_ID = "rawbytes";
public static final String TYPED_BYTES_ID = "typedbytes";
+ public static final String KEY_ONLY_TEXT_ID = "keyonlytext";
private Class<? extends InputWriter> inputWriterClass = null;
private Class<? extends OutputReader> outputReaderClass = null;
@@ -55,6 +57,11 @@ public class IdentifierResolver {
setOutputReaderClass(TypedBytesOutputReader.class);
setOutputKeyClass(TypedBytesWritable.class);
setOutputValueClass(TypedBytesWritable.class);
+ } else if (identifier.equalsIgnoreCase(KEY_ONLY_TEXT_ID)) {
+ setInputWriterClass(KeyOnlyTextInputWriter.class);
+ setOutputReaderClass(KeyOnlyTextOutputReader.class);
+ setOutputKeyClass(Text.class);
+ setOutputValueClass(NullWritable.class);
} else { // assume TEXT_ID
setInputWriterClass(TextInputWriter.class);
setOutputReaderClass(TextOutputReader.class);
Modified: hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/TextInputWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/TextInputWriter.java?rev=1534279&r1=1534278&r2=1534279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/TextInputWriter.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/io/TextInputWriter.java Mon Oct 21 17:11:34 2013
@@ -30,7 +30,7 @@ import org.apache.hadoop.streaming.PipeM
*/
public class TextInputWriter extends InputWriter<Object, Object> {
- private DataOutput clientOut;
+ protected DataOutput clientOut;
private byte[] inputSeparator;
@Override
@@ -53,7 +53,7 @@ public class TextInputWriter extends Inp
}
// Write an object to the output stream using UTF-8 encoding
- private void writeUTF8(Object object) throws IOException {
+ protected void writeUTF8(Object object) throws IOException {
byte[] bval;
int valSize;
if (object instanceof BytesWritable) {