You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2014/08/13 19:32:19 UTC
svn commit: r1617776 - in
/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/
src/main/java/org/apache/hadoop/fs/shell/ src/site/apt/ src/test/resources/
Author: aw
Date: Wed Aug 13 17:32:18 2014
New Revision: 1617776
URL: http://svn.apache.org/r1617776
Log:
HADOOP-8944. Shell command fs -count should include human readable option (Jonathan Allen via aw)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1617776&r1=1617775&r2=1617776&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Aug 13 17:32:18 2014
@@ -75,6 +75,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10281. Create a scheduler, which assigns schedulables a priority
level. (Chris Li via Arpit Agarwal)
+ HADOOP-8944. Shell command fs -count should include human readable option
+ (Jonathan Allen via aw)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java?rev=1617776&r1=1617775&r2=1617776&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java Wed Aug 13 17:32:18 2014
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.StringUtils;
/** Store the summary of a content (a directory or a file). */
@InterfaceAudience.Public
@@ -102,7 +103,7 @@ public class ContentSummary implements W
* <----12----> <----12----> <-------18------->
* DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME
*/
- private static final String STRING_FORMAT = "%12d %12d %18d ";
+ private static final String STRING_FORMAT = "%12s %12s %18s ";
/**
* Output format:
* <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18------->
@@ -117,7 +118,7 @@ public class ContentSummary implements W
private static final String QUOTA_HEADER = String.format(
QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
- "quota", "remaining quota", "space quota", "reamaining quota") +
+ "name quota", "rem name quota", "space quota", "rem space quota") +
HEADER;
/** Return the header of the output.
@@ -139,11 +140,25 @@ public class ContentSummary implements W
/** Return the string representation of the object in the output format.
* if qOption is false, output directory count, file count, and content size;
* if qOption is true, output quota and remaining quota as well.
+ *
+ * @param qOption a flag indicating if quota needs to be printed or not
+ * @return the string representation of the object
+ */
+ public String toString(boolean qOption) {
+ return toString(qOption, false);
+ }
+
+ /** Return the string representation of the object in the output format.
+ * if qOption is false, output directory count, file count, and content size;
+ * if qOption is true, output quota and remaining quota as well.
+ * if hOption is false file sizes are returned in bytes
+ * if hOption is true file sizes are returned in human readable
*
* @param qOption a flag indicating if quota needs to be printed or not
+ * @param hOption a flag indicating if human readable output if to be used
* @return the string representation of the object
*/
- public String toString(boolean qOption) {
+ public String toString(boolean qOption, boolean hOption) {
String prefix = "";
if (qOption) {
String quotaStr = "none";
@@ -152,19 +167,32 @@ public class ContentSummary implements W
String spaceQuotaRem = "inf";
if (quota>0) {
- quotaStr = Long.toString(quota);
- quotaRem = Long.toString(quota-(directoryCount+fileCount));
+ quotaStr = formatSize(quota, hOption);
+ quotaRem = formatSize(quota-(directoryCount+fileCount), hOption);
}
if (spaceQuota>0) {
- spaceQuotaStr = Long.toString(spaceQuota);
- spaceQuotaRem = Long.toString(spaceQuota - spaceConsumed);
+ spaceQuotaStr = formatSize(spaceQuota, hOption);
+ spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
}
prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem);
}
- return prefix + String.format(STRING_FORMAT, directoryCount,
- fileCount, length);
+ return prefix + String.format(STRING_FORMAT,
+ formatSize(directoryCount, hOption),
+ formatSize(fileCount, hOption),
+ formatSize(length, hOption));
+ }
+ /**
+ * Formats a size to be human readable or in bytes
+ * @param size value to be formatted
+ * @param humanReadable flag indicating human readable or not
+ * @return String representation of the size
+ */
+ private String formatSize(long size, boolean humanReadable) {
+ return humanReadable
+ ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
+ : String.valueOf(size);
}
}
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java?rev=1617776&r1=1617775&r2=1617776&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java Wed Aug 13 17:32:18 2014
@@ -42,16 +42,22 @@ public class Count extends FsCommand {
factory.addClass(Count.class, "-count");
}
+ private static final String OPTION_QUOTA = "q";
+ private static final String OPTION_HUMAN = "h";
+
public static final String NAME = "count";
- public static final String USAGE = "[-q] <path> ...";
+ public static final String USAGE =
+ "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
"that match the specified file pattern. The output columns are:\n" +
"DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" +
"QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" +
- " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME";
+ " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" +
+ "The -h option shows file sizes in human readable format.";
private boolean showQuotas;
+ private boolean humanReadable;
/** Constructor */
public Count() {}
@@ -70,17 +76,37 @@ public class Count extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args) {
- CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "q");
+ CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
+ OPTION_QUOTA, OPTION_HUMAN);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
args.add(".");
}
- showQuotas = cf.getOpt("q");
+ showQuotas = cf.getOpt(OPTION_QUOTA);
+ humanReadable = cf.getOpt(OPTION_HUMAN);
}
@Override
protected void processPath(PathData src) throws IOException {
ContentSummary summary = src.fs.getContentSummary(src.path);
- out.println(summary.toString(showQuotas) + src);
+ out.println(summary.toString(showQuotas, isHumanReadable()) + src);
+ }
+
+ /**
+ * Should quotas get shown as part of the report?
+ * @return if quotas should be shown then true otherwise false
+ */
+ @InterfaceAudience.Private
+ boolean isShowQuotas() {
+ return showQuotas;
+ }
+
+ /**
+ * Should sizes be shown in human readable format rather than bytes?
+ * @return true if human readable format
+ */
+ @InterfaceAudience.Private
+ boolean isHumanReadable() {
+ return humanReadable;
}
}
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm?rev=1617776&r1=1617775&r2=1617776&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm Wed Aug 13 17:32:18 2014
@@ -138,7 +138,7 @@ copyToLocal
count
- Usage: <<<hdfs dfs -count [-q] <paths> >>>
+ Usage: <<<hdfs dfs -count [-q] [-h] <paths> >>>
Count the number of directories, files and bytes under the paths that match
the specified file pattern. The output columns with -count are: DIR_COUNT,
@@ -147,12 +147,16 @@ count
The output columns with -count -q are: QUOTA, REMAINING_QUATA, SPACE_QUOTA,
REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, FILE_NAME
+ The -h option shows sizes in human readable format.
+
Example:
* <<<hdfs dfs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>>
* <<<hdfs dfs -count -q hdfs://nn1.example.com/file1>>>
+ * <<<hdfs dfs -count -q -h hdfs://nn1.example.com/file1>>>
+
Exit Code:
Returns 0 on success and -1 on error.
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml?rev=1617776&r1=1617775&r2=1617776&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml Wed Aug 13 17:32:18 2014
@@ -238,7 +238,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-count \[-q\] <path> \.\.\. :\s*</expected-output>
+ <expected-output>^-count \[-q\] \[-h\] <path> \.\.\. :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -260,6 +260,10 @@
<type>RegexpComparator</type>
<expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )*</expected-output>
</comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*The -h option shows file sizes in human readable format.( )*</expected-output>
+ </comparator>
</comparators>
</test>