You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ha...@apache.org on 2012/09/22 20:56:07 UTC
svn commit: r1388869 -
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
Author: harsh
Date: Sat Sep 22 18:56:07 2012
New Revision: 1388869
URL: http://svn.apache.org/viewvc?rev=1388869&view=rev
Log:
HADOOP-8833. fs -text should make sure to call inputstream.seek(0) before using input stream. Contributed by Tom White and Harsh J. (harsh)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java?rev=1388869&r1=1388868&r2=1388869&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java Sat Sep 22 18:56:07 2012
@@ -35,6 +35,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Random;
import java.util.Scanner;
+import java.util.zip.DeflaterOutputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.logging.Log;
@@ -52,7 +53,10 @@ import org.apache.hadoop.hdfs.tools.DFSA
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Test;
@@ -577,6 +581,8 @@ public class TestDFSShell {
try {
final FileSystem fs = root.getFileSystem(conf);
fs.mkdirs(root);
+
+ // Test the gzip type of files. Magic detection.
OutputStream zout = new GZIPOutputStream(
fs.create(new Path(root, "file.gz")));
Random r = new Random();
@@ -601,7 +607,7 @@ public class TestDFSShell {
Arrays.equals(file.toByteArray(), out.toByteArray()));
// Create a sequence file with a gz extension, to test proper
- // container detection
+ // container detection. Magic detection.
SequenceFile.Writer writer = SequenceFile.createWriter(
conf,
SequenceFile.Writer.file(new Path(root, "file.gz")),
@@ -619,6 +625,45 @@ public class TestDFSShell {
assertTrue("Output doesn't match input",
Arrays.equals("Foo\tBar\n".getBytes(), out.toByteArray()));
out.reset();
+
+ // Test deflate. Extension-based detection.
+ OutputStream dout = new DeflaterOutputStream(
+ fs.create(new Path(root, "file.deflate")));
+ byte[] outbytes = "foo".getBytes();
+ dout.write(outbytes);
+ dout.close();
+ out = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(out));
+ argv = new String[2];
+ argv[0] = "-text";
+ argv[1] = new Path(root, "file.deflate").toString();
+ ret = ToolRunner.run(new FsShell(conf), argv);
+ assertEquals("'-text " + argv[1] + " returned " + ret, 0, ret);
+ assertTrue("Output doesn't match input",
+ Arrays.equals(outbytes, out.toByteArray()));
+ out.reset();
+
+ // Test a simple codec. Extension based detection. We use
+ // Bzip2 cause its non-native.
+ CompressionCodec codec = (CompressionCodec)
+ ReflectionUtils.newInstance(BZip2Codec.class, conf);
+ String extension = codec.getDefaultExtension();
+ Path p = new Path(root, "file." + extension);
+ OutputStream fout = new DataOutputStream(codec.createOutputStream(
+ fs.create(p, true)));
+ byte[] writebytes = "foo".getBytes();
+ fout.write(writebytes);
+ fout.close();
+ out = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(out));
+ argv = new String[2];
+ argv[0] = "-text";
+ argv[1] = new Path(root, p).toString();
+ ret = ToolRunner.run(new FsShell(conf), argv);
+ assertEquals("'-text " + argv[1] + " returned " + ret, 0, ret);
+ assertTrue("Output doesn't match input",
+ Arrays.equals(writebytes, out.toByteArray()));
+ out.reset();
} finally {
if (null != bak) {
System.setOut(bak);