You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2015/04/14 06:19:29 UTC
hadoop git commit: HADOOP-7713. dfs -count -q should label output
column (Jonathan Allen via aw)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 7c072bf09 -> b7a052c84
HADOOP-7713. dfs -count -q should label output column (Jonathan Allen via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7a052c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7a052c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7a052c8
Branch: refs/heads/branch-2
Commit: b7a052c84af013c376e8c15f90202505571fc5d0
Parents: 7c072bf
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Feb 5 07:44:49 2015 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Apr 13 21:18:59 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 4 +-
.../org/apache/hadoop/fs/ContentSummary.java | 57 +++++++---
.../java/org/apache/hadoop/fs/shell/Count.java | 32 ++++--
.../apache/hadoop/fs/TestContentSummary.java | 6 +-
.../org/apache/hadoop/fs/shell/TestCount.java | 110 +++++++++++++++++--
.../src/test/resources/testConf.xml | 16 ++-
.../src/test/resources/testHDFSConf.xml | 42 +++++++
7 files changed, 226 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7a052c8/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index aff33ee..742a054 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -16,7 +16,6 @@ Release 2.8.0 - UNRELEASED
HADOOP-11719. [Fsshell] Remove bin/hadoop reference from
GenericOptionsParser default help text.
- (Brahma Reddy Battula via harsh)
HADOOP-11692. Improve authentication failure WARN message to avoid user
confusion. (Yongjun Zhang)
@@ -48,6 +47,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11814. Reformat hadoop-annotations, o.a.h.classification.tools.
(Li Lu via wheat9)
+ HADOOP-7713. dfs -count -q should label output column (Jonathan Allen
+ via aw)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7a052c8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
index dc4a7fa..66137d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
@@ -238,28 +238,35 @@ public class ContentSummary implements Writable{
this.spaceConsumed = in.readLong();
this.spaceQuota = in.readLong();
}
-
- /**
+
+ /**
* Output format:
* <----12----> <----12----> <-------18------->
- * DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME
+ * DIR_COUNT FILE_COUNT CONTENT_SIZE
*/
- private static final String STRING_FORMAT = "%12s %12s %18s ";
- /**
+ private static final String SUMMARY_FORMAT = "%12s %12s %18s ";
+ /**
* Output format:
- * <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18------->
- * QUOTA REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME
+ * <----12----> <------15-----> <------15-----> <------15----->
+ * QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA
+ * <----12----> <----12----> <-------18------->
+ * DIR_COUNT FILE_COUNT CONTENT_SIZE
*/
- private static final String QUOTA_STRING_FORMAT = "%12s %15s ";
- private static final String SPACE_QUOTA_STRING_FORMAT = "%15s %15s ";
-
+ private static final String QUOTA_SUMMARY_FORMAT = "%12s %15s ";
+ private static final String SPACE_QUOTA_SUMMARY_FORMAT = "%15s %15s ";
+
+ private static final String[] HEADER_FIELDS = new String[] { "DIR_COUNT",
+ "FILE_COUNT", "CONTENT_SIZE"};
+ private static final String[] QUOTA_HEADER_FIELDS = new String[] { "QUOTA",
+ "REM_QUOTA", "SPACE_QUOTA", "REM_SPACE_QUOTA" };
+
/** The header string */
private static final String HEADER = String.format(
- STRING_FORMAT.replace('d', 's'), "directories", "files", "bytes");
+ SUMMARY_FORMAT, (Object[]) HEADER_FIELDS);
private static final String QUOTA_HEADER = String.format(
- QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
- "name quota", "rem name quota", "space quota", "rem space quota") +
+ QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
+ (Object[]) QUOTA_HEADER_FIELDS) +
HEADER;
/** Return the header of the output.
@@ -272,7 +279,25 @@ public class ContentSummary implements Writable{
public static String getHeader(boolean qOption) {
return qOption ? QUOTA_HEADER : HEADER;
}
-
+
+ /**
+ * Returns the names of the fields from the summary header.
+ *
+ * @return names of fields as displayed in the header
+ */
+ public static String[] getHeaderFields() {
+ return HEADER_FIELDS;
+ }
+
+ /**
+ * Returns the names of the fields used in the quota summary.
+ *
+ * @return names of quota fields as displayed in the header
+ */
+ public static String[] getQuotaHeaderFields() {
+ return QUOTA_HEADER_FIELDS;
+ }
+
@Override
public String toString() {
return toString(true);
@@ -316,11 +341,11 @@ public class ContentSummary implements Writable{
spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
}
- prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
+ prefix = String.format(QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem);
}
- return prefix + String.format(STRING_FORMAT,
+ return prefix + String.format(SUMMARY_FORMAT,
formatSize(directoryCount, hOption),
formatSize(fileCount, hOption),
formatSize(length, hOption));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7a052c8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
index ff7a10f..dd7d168 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.LinkedList;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -44,18 +45,26 @@ public class Count extends FsCommand {
private static final String OPTION_QUOTA = "q";
private static final String OPTION_HUMAN = "h";
+ private static final String OPTION_HEADER = "v";
public static final String NAME = "count";
public static final String USAGE =
- "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] <path> ...";
- public static final String DESCRIPTION =
+ "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ + "] <path> ...";
+ public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
- "that match the specified file pattern. The output columns are:\n" +
- "DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" +
- "QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" +
- " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" +
- "The -h option shows file sizes in human readable format.";
-
+ "that match the specified file pattern. The output columns are:\n" +
+ StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
+ " PATHNAME\n" +
+ "or, with the -" + OPTION_QUOTA + " option:\n" +
+ StringUtils.join(ContentSummary.getQuotaHeaderFields(), ' ') + "\n" +
+ " " +
+ StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
+ " PATHNAME\n" +
+ "The -" + OPTION_HUMAN +
+ " option shows file sizes in human readable format.\n" +
+ "The -" + OPTION_HEADER + " option displays a header line.";
+
private boolean showQuotas;
private boolean humanReadable;
@@ -65,7 +74,7 @@ public class Count extends FsCommand {
/** Constructor
* @deprecated invoke via {@link FsShell}
* @param cmd the count command
- * @param pos the starting index of the arguments
+ * @param pos the starting index of the arguments
* @param conf configuration
*/
@Deprecated
@@ -77,13 +86,16 @@ public class Count extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args) {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
- OPTION_QUOTA, OPTION_HUMAN);
+ OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
args.add(".");
}
showQuotas = cf.getOpt(OPTION_QUOTA);
humanReadable = cf.getOpt(OPTION_HUMAN);
+ if (cf.getOpt(OPTION_HEADER)) {
+ out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME");
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7a052c8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
index b0ebe57..7cc7ae4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
@@ -140,15 +140,15 @@ public class TestContentSummary {
// check the header with quotas
@Test
public void testGetHeaderWithQuota() {
- String header = " name quota rem name quota space quota "
- + "rem space quota directories files bytes ";
+ String header = " QUOTA REM_QUOTA SPACE_QUOTA "
+ + "REM_SPACE_QUOTA DIR_COUNT FILE_COUNT CONTENT_SIZE ";
assertEquals(header, ContentSummary.getHeader(true));
}
// check the header without quotas
@Test
public void testGetHeaderNoQuota() {
- String header = " directories files bytes ";
+ String header = " DIR_COUNT FILE_COUNT CONTENT_SIZE ";
assertEquals(header, ContentSummary.getHeader(false));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7a052c8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
index 027b55f..11173b3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.CommandFormat.NotEnoughArgumentsException;
import org.junit.Test;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -47,7 +48,6 @@ public class TestCount {
private static Configuration conf;
private static FileSystem mockFs;
private static FileStatus fileStat;
- private static ContentSummary mockCs;
@BeforeClass
public static void setup() {
@@ -55,7 +55,6 @@ public class TestCount {
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
mockFs = mock(FileSystem.class);
fileStat = mock(FileStatus.class);
- mockCs = mock(ContentSummary.class);
when(fileStat.isFile()).thenReturn(true);
}
@@ -87,6 +86,85 @@ public class TestCount {
assertTrue(count.isHumanReadable());
}
+ // check no options is handled correctly
+ @Test
+ public void processOptionsNoOptions() {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("dummy");
+ Count count = new Count();
+ count.processOptions(options);
+ assertFalse(count.isShowQuotas());
+ }
+
+ // check -q is handled correctly
+ @Test
+ public void processOptionsShowQuotas() {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-q");
+ options.add("dummy");
+ Count count = new Count();
+ count.processOptions(options);
+ assertTrue(count.isShowQuotas());
+ }
+
+ // check missing arguments is handled correctly
+ @Test
+ public void processOptionsMissingArgs() {
+ LinkedList<String> options = new LinkedList<String>();
+ Count count = new Count();
+ try {
+ count.processOptions(options);
+ fail("Count.processOptions - NotEnoughArgumentsException not thrown");
+ } catch (NotEnoughArgumentsException e) {
+ }
+ assertFalse(count.isShowQuotas());
+ }
+
+ // check the correct header is produced with no quotas (-v)
+ @Test
+ public void processOptionsHeaderNoQuotas() {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-v");
+ options.add("dummy");
+
+ PrintStream out = mock(PrintStream.class);
+
+ Count count = new Count();
+ count.out = out;
+
+ count.processOptions(options);
+
+ String noQuotasHeader =
+ // <----12----> <----12----> <-------18------->
+ " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME";
+ verify(out).println(noQuotasHeader);
+ verifyNoMoreInteractions(out);
+ }
+
+ // check the correct header is produced with quotas (-q -v)
+ @Test
+ public void processOptionsHeaderWithQuotas() {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-q");
+ options.add("-v");
+ options.add("dummy");
+
+ PrintStream out = mock(PrintStream.class);
+
+ Count count = new Count();
+ count.out = out;
+
+ count.processOptions(options);
+
+ String withQuotasHeader =
+ // <----12----> <-----15------> <-----15------> <-----15------>
+ " QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA " +
+ // <----12----> <----12----> <-------18------->
+ " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME";
+ verify(out).println(withQuotasHeader);
+ verifyNoMoreInteractions(out);
+ }
+
// check quotas are reported correctly
@Test
public void processPathShowQuotas() throws Exception {
@@ -211,10 +289,28 @@ public class TestCount {
public void getUsage() {
Count count = new Count();
String actual = count.getUsage();
- String expected = "-count [-q] [-h] <path> ...";
+ String expected = "-count [-q] [-h] [-v] <path> ...";
assertEquals("Count.getUsage", expected, actual);
}
+ // check the correct description is returned
+ @Test
+ public void getDescription() {
+ Count count = new Count();
+ String actual = count.getDescription();
+ String expected =
+ "Count the number of directories, files and bytes under the paths\n"
+ + "that match the specified file pattern. The output columns are:\n"
+ + "DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ + "or, with the -q option:\n"
+ + "QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA\n"
+ + " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ + "The -h option shows file sizes in human readable format.\n"
+ + "The -v option displays a header line.";
+
+ assertEquals("Count.getDescription", expected, actual);
+ }
+
// mock content system
static class MockContentSummary extends ContentSummary {
@@ -227,15 +323,15 @@ public class TestCount {
public String toString(boolean qOption, boolean hOption) {
if (qOption) {
if (hOption) {
- return(HUMAN + WITH_QUOTAS);
+ return (HUMAN + WITH_QUOTAS);
} else {
- return(BYTES + WITH_QUOTAS);
+ return (BYTES + WITH_QUOTAS);
}
} else {
if (hOption) {
- return(HUMAN + NO_QUOTAS);
+ return (HUMAN + NO_QUOTAS);
} else {
- return(BYTES + NO_QUOTAS);
+ return (BYTES + NO_QUOTAS);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7a052c8/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 52da00b..36e138a 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -238,7 +238,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-count \[-q\] \[-h\] <path> \.\.\. :( )*</expected-output>
+ <expected-output>^-count \[-q\] \[-h\] \[-v\] <path> \.\.\. :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -250,20 +250,28 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or( )*</expected-output>
+ <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA( )*</expected-output>
+ <expected-output>^( |\t)*or, with the -q option:( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )*</expected-output>
+ <expected-output>^( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^( |\t)*The -h option shows file sizes in human readable format.( )*</expected-output>
</comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output>
+ </comparator>
</comparators>
</test>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7a052c8/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index 8e7af92..ac26ce5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -8699,6 +8699,48 @@
</comparators>
</test>
+ <test> <!-- TESTED -->
+ <description>count: file using absolute path showing header record</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -count -v /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
+ <description>count: file using absolute path with -q option and showing header record</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -count -q -v /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>( |\t)*none( |\t)*inf( |\t)*none( |\t)*inf( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
<!-- Tests for chmod -->
<test> <!-- TESTED -->
<description>chmod: change permission(octal mode) of file in absolute path</description>