You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by je...@apache.org on 2013/03/07 00:33:10 UTC
svn commit: r1453620 - in
/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common:
CHANGES.txt src/main/java/org/apache/hadoop/fs/shell/Display.java
src/test/resources/testConf.xml
Author: jeagles
Date: Wed Mar 6 23:33:09 2013
New Revision: 1453620
URL: http://svn.apache.org/r1453620
Log:
HADOOP-9209. Add shell command to dump file checksums (Todd Lipcon via jeagles)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1453620&r1=1453619&r2=1453620&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Mar 6 23:33:09 2013
@@ -1072,6 +1072,9 @@ Release 0.23.7 - UNRELEASED
HADOOP-9352. Expose UGI.setLoginUser for tests (daryn)
+ HADOOP-9209. Add shell command to dump file checksums (Todd Lipcon via
+ jeagles)
+
OPTIMIZATIONS
HADOOP-8462. Native-code implementation of bzip2 codec. (Govind Kamat via
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java?rev=1453620&r1=1453619&r2=1453620&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java Wed Mar 6 23:33:09 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsDirectoryException;
@@ -48,13 +49,14 @@ import org.apache.hadoop.io.WritableComp
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
import org.codehaus.jackson.JsonEncoding;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.util.MinimalPrettyPrinter;
/**
- * Display contents of files
+ * Display contents or checksums of files
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -63,6 +65,7 @@ class Display extends FsCommand {
public static void registerCommands(CommandFactory factory) {
factory.addClass(Cat.class, "-cat");
factory.addClass(Text.class, "-text");
+ factory.addClass(Checksum.class, "-checksum");
}
/**
@@ -162,6 +165,36 @@ class Display extends FsCommand {
return i;
}
}
+
+ public static class Checksum extends Display {
+ public static final String NAME = "checksum";
+ public static final String USAGE = "<src> ...";
+ public static final String DESCRIPTION =
+ "Dump checksum information for files that match the file\n" +
+ "pattern <src> to stdout. Note that this requires a round-trip\n" +
+ "to a datanode storing each block of the file, and thus is not\n" +
+ "efficient to run on a large number of files. The checksum of a\n" +
+ "file depends on its content, block size and the checksum\n" +
+ "algorithm and parameters used for creating the file.";
+
+ @Override
+ protected void processPath(PathData item) throws IOException {
+ if (item.stat.isDirectory()) {
+ throw new PathIsDirectoryException(item.toString());
+ }
+
+ FileChecksum checksum = item.fs.getFileChecksum(item.path);
+ if (checksum == null) {
+ out.printf("%s\tNONE\t\n", item.toString());
+ } else {
+ String checksumString = StringUtils.byteToHexString(
+ checksum.getBytes(), 0, checksum.getLength());
+ out.printf("%s\t%s\t%s\n",
+ item.toString(), checksum.getAlgorithmName(),
+ checksumString);
+ }
+ }
+ }
protected class TextRecordInputStream extends InputStream {
SequenceFile.Reader r;
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml?rev=1453620&r1=1453619&r2=1453620&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml Wed Mar 6 23:33:09 2013
@@ -487,6 +487,22 @@
</test>
<test> <!-- TESTED -->
+ <description>help: help for checksum</description>
+ <test-commands>
+ <command>-help checksum</command>
+ </test-commands>
+ <cleanup-commands>
+ <!-- No cleanup -->
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^-checksum <src> \.\.\.:( |\t)*Dump checksum information for files.*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
<description>help: help for copyToLocal</description>
<test-commands>
<command>-help copyToLocal</command>