You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2016/08/23 17:32:22 UTC
[18/50] [abbrv] hadoop git commit: HDFS-8986. Add option to -du to
calculate directory space usage excluding snapshots. Contributed by Xiao
Chen.
HDFS-8986. Add option to -du to calculate directory space usage excluding snapshots. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0efea49
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0efea49
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0efea49
Branch: refs/heads/YARN-3368
Commit: f0efea490e5aa9dd629d2199aae9c5b1290a17ee
Parents: dd76238
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Tue Aug 23 04:13:48 2016 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Tue Aug 23 04:14:24 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/ContentSummary.java | 127 ++++++++--
.../java/org/apache/hadoop/fs/shell/Count.java | 20 +-
.../org/apache/hadoop/fs/shell/FsUsage.java | 31 ++-
.../src/site/markdown/FileSystemShell.md | 11 +-
.../org/apache/hadoop/fs/shell/TestCount.java | 5 +-
.../src/test/resources/testConf.xml | 12 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 8 +
.../src/main/proto/hdfs.proto | 4 +
.../ContentSummaryComputationContext.java | 6 +
.../hadoop/hdfs/server/namenode/INode.java | 9 +-
.../hdfs/server/namenode/INodeDirectory.java | 4 +
.../org/apache/hadoop/hdfs/TestDFSShell.java | 235 ++++++++++++++++++-
12 files changed, 432 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
index 3dedbcc..3e75951 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
@@ -34,6 +34,11 @@ public class ContentSummary extends QuotaUsage implements Writable{
private long length;
private long fileCount;
private long directoryCount;
+ // These fields are to track the snapshot-related portion of the values.
+ private long snapshotLength;
+ private long snapshotFileCount;
+ private long snapshotDirectoryCount;
+ private long snapshotSpaceConsumed;
/** We don't use generics. Instead override spaceConsumed and other methods
in order to keep backward compatibility. */
@@ -56,6 +61,26 @@ public class ContentSummary extends QuotaUsage implements Writable{
return this;
}
+ public Builder snapshotLength(long snapshotLength) {
+ this.snapshotLength = snapshotLength;
+ return this;
+ }
+
+ public Builder snapshotFileCount(long snapshotFileCount) {
+ this.snapshotFileCount = snapshotFileCount;
+ return this;
+ }
+
+ public Builder snapshotDirectoryCount(long snapshotDirectoryCount) {
+ this.snapshotDirectoryCount = snapshotDirectoryCount;
+ return this;
+ }
+
+ public Builder snapshotSpaceConsumed(long snapshotSpaceConsumed) {
+ this.snapshotSpaceConsumed = snapshotSpaceConsumed;
+ return this;
+ }
+
@Override
public Builder quota(long quota){
super.quota(quota);
@@ -107,6 +132,10 @@ public class ContentSummary extends QuotaUsage implements Writable{
private long length;
private long fileCount;
private long directoryCount;
+ private long snapshotLength;
+ private long snapshotFileCount;
+ private long snapshotDirectoryCount;
+ private long snapshotSpaceConsumed;
}
/** Constructor deprecated by ContentSummary.Builder*/
@@ -142,17 +171,37 @@ public class ContentSummary extends QuotaUsage implements Writable{
this.length = builder.length;
this.fileCount = builder.fileCount;
this.directoryCount = builder.directoryCount;
+ this.snapshotLength = builder.snapshotLength;
+ this.snapshotFileCount = builder.snapshotFileCount;
+ this.snapshotDirectoryCount = builder.snapshotDirectoryCount;
+ this.snapshotSpaceConsumed = builder.snapshotSpaceConsumed;
}
/** @return the length */
public long getLength() {return length;}
+ public long getSnapshotLength() {
+ return snapshotLength;
+ }
+
/** @return the directory count */
public long getDirectoryCount() {return directoryCount;}
+ public long getSnapshotDirectoryCount() {
+ return snapshotDirectoryCount;
+ }
+
/** @return the file count */
public long getFileCount() {return fileCount;}
+ public long getSnapshotFileCount() {
+ return snapshotFileCount;
+ }
+
+ public long getSnapshotSpaceConsumed() {
+ return snapshotSpaceConsumed;
+ }
+
@Override
@InterfaceAudience.Private
public void write(DataOutput out) throws IOException {
@@ -180,9 +229,14 @@ public class ContentSummary extends QuotaUsage implements Writable{
if (this == to) {
return true;
} else if (to instanceof ContentSummary) {
- return getLength() == ((ContentSummary) to).getLength() &&
- getFileCount() == ((ContentSummary) to).getFileCount() &&
- getDirectoryCount() == ((ContentSummary) to).getDirectoryCount() &&
+ ContentSummary right = (ContentSummary) to;
+ return getLength() == right.getLength() &&
+ getFileCount() == right.getFileCount() &&
+ getDirectoryCount() == right.getDirectoryCount() &&
+ getSnapshotLength() == right.getSnapshotLength() &&
+ getSnapshotFileCount() == right.getSnapshotFileCount() &&
+ getSnapshotDirectoryCount() == right.getSnapshotDirectoryCount() &&
+ getSnapshotSpaceConsumed() == right.getSnapshotSpaceConsumed() &&
super.equals(to);
} else {
return super.equals(to);
@@ -191,7 +245,9 @@ public class ContentSummary extends QuotaUsage implements Writable{
@Override
public int hashCode() {
- long result = getLength() ^ getFileCount() ^ getDirectoryCount();
+ long result = getLength() ^ getFileCount() ^ getDirectoryCount()
+ ^ getSnapshotLength() ^ getSnapshotFileCount()
+ ^ getSnapshotDirectoryCount() ^ getSnapshotSpaceConsumed();
return ((int) result) ^ super.hashCode();
}
@@ -255,15 +311,14 @@ public class ContentSummary extends QuotaUsage implements Writable{
* @param qOption a flag indicating if quota needs to be printed or not
* @return the string representation of the object
*/
+ @Override
public String toString(boolean qOption) {
return toString(qOption, false);
}
/** Return the string representation of the object in the output format.
- * if qOption is false, output directory count, file count, and content size;
- * if qOption is true, output quota and remaining quota as well.
- * if hOption is false file sizes are returned in bytes
- * if hOption is true file sizes are returned in human readable
+ * For description of the options,
+ * @see #toString(boolean, boolean, boolean, boolean, List)
*
* @param qOption a flag indicating if quota needs to be printed or not
* @param hOption a flag indicating if human readable output if to be used
@@ -273,10 +328,24 @@ public class ContentSummary extends QuotaUsage implements Writable{
return toString(qOption, hOption, false, null);
}
+ /** Return the string representation of the object in the output format.
+ * For description of the options,
+ * @see #toString(boolean, boolean, boolean, boolean, List)
+ *
+ * @param qOption a flag indicating if quota needs to be printed or not
+ * @param hOption a flag indicating if human readable output is to be used
+ * @param xOption a flag indicating if calculation from snapshots is to be
+ * included in the output
+ * @return the string representation of the object
+ */
+ public String toString(boolean qOption, boolean hOption, boolean xOption) {
+ return toString(qOption, hOption, false, xOption, null);
+ }
+
/**
* Return the string representation of the object in the output format.
- * if tOption is true, display the quota by storage types,
- * Otherwise, same logic with #toString(boolean,boolean)
+ * For description of the options,
+ * @see #toString(boolean, boolean, boolean, boolean, List)
*
* @param qOption a flag indicating if quota needs to be printed or not
* @param hOption a flag indicating if human readable output if to be used
@@ -286,6 +355,29 @@ public class ContentSummary extends QuotaUsage implements Writable{
*/
public String toString(boolean qOption, boolean hOption,
boolean tOption, List<StorageType> types) {
+ return toString(qOption, hOption, tOption, false, types);
+ }
+
+ /** Return the string representation of the object in the output format.
+ * if qOption is false, output directory count, file count, and content size;
+ * if qOption is true, output quota and remaining quota as well.
+ * if hOption is false, file sizes are returned in bytes
+ * if hOption is true, file sizes are returned in human readable
+ * if tOption is true, display the quota by storage types
+ * if tOption is false, same logic with #toString(boolean,boolean)
+ * if xOption is false, output includes the calculation from snapshots
+ * if xOption is true, output excludes the calculation from snapshots
+ *
+ * @param qOption a flag indicating if quota needs to be printed or not
+ * @param hOption a flag indicating if human readable output is to be used
+ * @param tOption a flag indicating if display quota by storage types
+ * @param xOption a flag indicating if calculation from snapshots is to be
+ * included in the output
+ * @param types Storage types to display
+ * @return the string representation of the object
+ */
+ public String toString(boolean qOption, boolean hOption, boolean tOption,
+ boolean xOption, List<StorageType> types) {
String prefix = "";
if (tOption) {
@@ -296,10 +388,17 @@ public class ContentSummary extends QuotaUsage implements Writable{
prefix = getQuotaUsage(hOption);
}
- return prefix + String.format(SUMMARY_FORMAT,
- formatSize(directoryCount, hOption),
- formatSize(fileCount, hOption),
- formatSize(length, hOption));
+ if (xOption) {
+ return prefix + String.format(SUMMARY_FORMAT,
+ formatSize(directoryCount - snapshotDirectoryCount, hOption),
+ formatSize(fileCount - snapshotFileCount, hOption),
+ formatSize(length - snapshotLength, hOption));
+ } else {
+ return prefix + String.format(SUMMARY_FORMAT,
+ formatSize(directoryCount, hOption),
+ formatSize(fileCount, hOption),
+ formatSize(length, hOption));
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
index 2b5a4cc..16f52e3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
@@ -51,6 +51,8 @@ public class Count extends FsCommand {
private static final String OPTION_HUMAN = "h";
private static final String OPTION_HEADER = "v";
private static final String OPTION_TYPE = "t";
+ // exclude snapshots from calculation. Only work on default columns.
+ private static final String OPTION_EXCLUDE_SNAPSHOT = "x";
//return the quota, namespace count and disk space usage.
private static final String OPTION_QUOTA_AND_USAGE = "u";
@@ -58,7 +60,8 @@ public class Count extends FsCommand {
public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ "] [-" + OPTION_TYPE + " [<storage type>]] [-" +
- OPTION_QUOTA_AND_USAGE + "] <path> ...";
+ OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT
+ + "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
"that match the specified file pattern. The output columns are:\n" +
@@ -72,6 +75,8 @@ public class Count extends FsCommand {
"The -" + OPTION_HUMAN +
" option shows file sizes in human readable format.\n" +
"The -" + OPTION_HEADER + " option displays a header line.\n" +
+ "The -" + OPTION_EXCLUDE_SNAPSHOT + " option excludes snapshots " +
+ "from being calculated. \n" +
"The -" + OPTION_TYPE + " option displays quota by storage types.\n" +
"It must be used with -" + OPTION_QUOTA + " option.\n" +
"If a comma-separated list of storage types is given after the -" +
@@ -87,6 +92,7 @@ public class Count extends FsCommand {
private boolean showQuotabyType;
private List<StorageType> storageTypes = null;
private boolean showQuotasAndUsageOnly;
+ private boolean excludeSnapshots;
/** Constructor */
public Count() {}
@@ -106,7 +112,8 @@ public class Count extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args) {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
- OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE);
+ OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE,
+ OPTION_EXCLUDE_SNAPSHOT);
cf.addOptionWithValue(OPTION_TYPE);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
@@ -115,6 +122,7 @@ public class Count extends FsCommand {
showQuotas = cf.getOpt(OPTION_QUOTA);
humanReadable = cf.getOpt(OPTION_HUMAN);
showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE);
+ excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT);
if (showQuotas || showQuotasAndUsageOnly) {
String types = cf.getOptValue(OPTION_TYPE);
@@ -125,6 +133,11 @@ public class Count extends FsCommand {
} else {
showQuotabyType = false;
}
+ if (excludeSnapshots) {
+ out.println(OPTION_QUOTA + " or " + OPTION_QUOTA_AND_USAGE + " option "
+ + "is given, the -x option is ignored.");
+ excludeSnapshots = false;
+ }
}
if (cf.getOpt(OPTION_HEADER)) {
@@ -163,7 +176,8 @@ public class Count extends FsCommand {
storageTypes) + src);
} else {
ContentSummary summary = src.fs.getContentSummary(src.path);
- out.println(summary.toString(showQuotas, isHumanReadable()) + src);
+ out.println(summary.
+ toString(showQuotas, isHumanReadable(), excludeSnapshots) + src);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
index 765b181..197920f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
@@ -107,27 +107,30 @@ class FsUsage extends FsCommand {
/** show disk usage */
public static class Du extends FsUsage {
public static final String NAME = "du";
- public static final String USAGE = "[-s] [-h] <path> ...";
+ public static final String USAGE = "[-s] [-h] [-x] <path> ...";
public static final String DESCRIPTION =
- "Show the amount of space, in bytes, used by the files that " +
- "match the specified file pattern. The following flags are optional:\n" +
- "-s: Rather than showing the size of each individual file that" +
- " matches the pattern, shows the total (summary) size.\n" +
- "-h: Formats the sizes of files in a human-readable fashion" +
- " rather than a number of bytes.\n\n" +
- "Note that, even without the -s option, this only shows size summaries " +
- "one level deep into a directory.\n\n" +
- "The output is in the form \n" +
- "\tsize\tdisk space consumed\tname(full path)\n";
+ "Show the amount of space, in bytes, used by the files that match " +
+ "the specified file pattern. The following flags are optional:\n" +
+ "-s: Rather than showing the size of each individual file that" +
+ " matches the pattern, shows the total (summary) size.\n" +
+ "-h: Formats the sizes of files in a human-readable fashion" +
+ " rather than a number of bytes.\n" +
+ "-x: Excludes snapshots from being counted.\n\n" +
+ "Note that, even without the -s option, this only shows size " +
+ "summaries one level deep into a directory.\n\n" +
+ "The output is in the form \n" +
+ "\tsize\tdisk space consumed\tname(full path)\n";
protected boolean summary = false;
+ private boolean excludeSnapshots = false;
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
- CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s");
+ CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s", "x");
cf.parse(args);
humanReadable = cf.getOpt("h");
summary = cf.getOpt("s");
+ excludeSnapshots = cf.getOpt("x");
if (args.isEmpty()) args.add(Path.CUR_DIR);
}
@@ -156,6 +159,10 @@ class FsUsage extends FsCommand {
ContentSummary contentSummary = item.fs.getContentSummary(item.path);
long length = contentSummary.getLength();
long spaceConsumed = contentSummary.getSpaceConsumed();
+ if (excludeSnapshots) {
+ length -= contentSummary.getSnapshotLength();
+ spaceConsumed -= contentSummary.getSnapshotSpaceConsumed();
+ }
usagesTable.addRow(formatSize(length), formatSize(spaceConsumed), item);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 066cfe3..a8a1920 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -132,10 +132,12 @@ Similar to get command, except that the destination is restricted to a local fil
count
-----
-Usage: `hadoop fs -count [-q] [-h] [-v] [-t [<storage type>]] [-u] <paths> `
+Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] <paths> `
Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
+The -u and -q options control what columns the output contains. -q means show quotas, -u limits the output to show quotas and usage only.
+
The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
The output columns with -count -u are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA
@@ -146,6 +148,8 @@ The -h option shows sizes in human readable format.
The -v option displays a header line.
+The -x option excludes snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path. The -x option is ignored if -u or -q option is given.
+
Example:
* `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
@@ -211,14 +215,15 @@ Example:
du
----
-Usage: `hadoop fs -du [-s] [-h] URI [URI ...]`
+Usage: `hadoop fs -du [-s] [-h] [-x] URI [URI ...]`
Displays sizes of files and directories contained in the given directory or the length of a file in case its just a file.
Options:
-* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files.
+* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files. Without the -s option, calculation is done by going 1-level deep from the given path.
* The -h option will format file sizes in a "human-readable" fashion (e.g 64.0m instead of 67108864)
+* The -x option will exclude snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path.
The du returns three columns with the following format:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
index 04174e0..c7b30e4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
@@ -447,7 +447,7 @@ public class TestCount {
Count count = new Count();
String actual = count.getUsage();
String expected =
- "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] <path> ...";
+ "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] <path> ...";
assertEquals("Count.getUsage", expected, actual);
}
@@ -465,6 +465,7 @@ public class TestCount {
+ " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ "The -h option shows file sizes in human readable format.\n"
+ "The -v option displays a header line.\n"
+ + "The -x option excludes snapshots from being calculated. \n"
+ "The -t option displays quota by storage types.\n"
+ "It must be used with -q option.\n"
+ "If a comma-separated list of storage types is given after the -t option, \n"
@@ -521,7 +522,7 @@ public class TestCount {
}
@Override
- public String toString(boolean qOption, boolean hOption) {
+ public String toString(boolean qOption, boolean hOption, boolean xOption) {
if (qOption) {
if (hOption) {
return (HUMAN + WITH_QUOTAS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 82bc789..a31c828 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -200,7 +200,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-du \[-s\] \[-h\] <path> \.\.\. :\s*</expected-output>
+ <expected-output>^-du \[-s\] \[-h\] \[-x\] <path> \.\.\. :\s*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -228,6 +228,10 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
+ <expected-output>^\s*-x\s*Excludes snapshots from being counted.\s*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
<expected-output>^\s*Note that, even without the -s option, this only shows size summaries one level\s*</expected-output>
</comparator>
<comparator>
@@ -274,7 +278,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] <path> \.\.\. :( )*</expected-output>
+ <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] \[-x\] <path> \.\.\. :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -308,6 +312,10 @@
<type>RegexpComparator</type>
<expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output>
</comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*The -x option excludes snapshots from being calculated.( )*</expected-output>
+ </comparator>
</comparators>
</test>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 3c5f583..7df68e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -1466,6 +1466,10 @@ public class PBHelperClient {
builder.length(cs.getLength()).
fileCount(cs.getFileCount()).
directoryCount(cs.getDirectoryCount()).
+ snapshotLength(cs.getSnapshotLength()).
+ snapshotFileCount(cs.getSnapshotFileCount()).
+ snapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
+ snapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
quota(cs.getQuota()).
spaceConsumed(cs.getSpaceConsumed()).
spaceQuota(cs.getSpaceQuota());
@@ -2069,6 +2073,10 @@ public class PBHelperClient {
builder.setLength(cs.getLength()).
setFileCount(cs.getFileCount()).
setDirectoryCount(cs.getDirectoryCount()).
+ setSnapshotLength(cs.getSnapshotLength()).
+ setSnapshotFileCount(cs.getSnapshotFileCount()).
+ setSnapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
+ setSnapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
setQuota(cs.getQuota()).
setSpaceConsumed(cs.getSpaceConsumed()).
setSpaceQuota(cs.getSpaceQuota());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index f939ec3..7de8550 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -138,6 +138,10 @@ message ContentSummaryProto {
required uint64 spaceConsumed = 5;
required uint64 spaceQuota = 6;
optional StorageTypeQuotaInfosProto typeQuotaInfos = 7;
+ optional uint64 snapshotLength = 8;
+ optional uint64 snapshotFileCount = 9;
+ optional uint64 snapshotDirectoryCount = 10;
+ optional uint64 snapshotSpaceConsumed = 11;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
index 5739835..6df9e75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
@@ -29,6 +29,7 @@ public class ContentSummaryComputationContext {
private FSNamesystem fsn = null;
private BlockStoragePolicySuite bsps = null;
private ContentCounts counts = null;
+ private ContentCounts snapshotCounts = null;
private long nextCountLimit = 0;
private long limitPerRun = 0;
private long yieldCount = 0;
@@ -51,6 +52,7 @@ public class ContentSummaryComputationContext {
this.limitPerRun = limitPerRun;
this.nextCountLimit = limitPerRun;
this.counts = new ContentCounts.Builder().build();
+ this.snapshotCounts = new ContentCounts.Builder().build();
this.sleepMilliSec = sleepMicroSec/1000;
this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000);
}
@@ -125,6 +127,10 @@ public class ContentSummaryComputationContext {
return counts;
}
+ public ContentCounts getSnapshotCounts() {
+ return snapshotCounts;
+ }
+
public BlockStoragePolicySuite getBlockStoragePolicySuite() {
Preconditions.checkState((bsps != null || fsn != null),
"BlockStoragePolicySuite must be either initialized or available via" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 0348c1f..c6258a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -428,8 +428,9 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
*/
public final ContentSummary computeAndConvertContentSummary(int snapshotId,
ContentSummaryComputationContext summary) {
- ContentCounts counts = computeContentSummary(snapshotId, summary)
- .getCounts();
+ computeContentSummary(snapshotId, summary);
+ final ContentCounts counts = summary.getCounts();
+ final ContentCounts snapshotCounts = summary.getSnapshotCounts();
final QuotaCounts q = getQuotaCounts();
return new ContentSummary.Builder().
length(counts.getLength()).
@@ -440,6 +441,10 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
spaceQuota(q.getStorageSpace()).
typeConsumed(counts.getTypeSpaces()).
typeQuota(q.getTypeSpaces().asArray()).
+ snapshotLength(snapshotCounts.getLength()).
+ snapshotFileCount(snapshotCounts.getFileCount()).
+ snapshotDirectoryCount(snapshotCounts.getDirectoryCount()).
+ snapshotSpaceConsumed(snapshotCounts.getStoragespace()).
build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
index d00c136..9a8f9b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
@@ -634,6 +634,10 @@ public class INodeDirectory extends INodeWithAdditionalFields
// computation should include all the deleted files/directories
sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
summary.getCounts());
+ // Also compute ContentSummary for snapshotCounts (So we can extract it
+ // later from the ContentSummary of all).
+ sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
+ summary.getSnapshotCounts());
}
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
if (q != null && snapshotId == Snapshot.CURRENT_STATE_ID) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index dcf683e..73ebf28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -113,6 +113,11 @@ public class TestDFSShell {
return p;
}
+ static void rmr(FileSystem fs, Path p) throws IOException {
+ assertTrue(fs.delete(p, true));
+ assertFalse(fs.exists(p));
+ }
+
static File createLocalFile(File f) throws IOException {
assertTrue(!f.exists());
PrintWriter out = new PrintWriter(f);
@@ -189,7 +194,7 @@ public class TestDFSShell {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
FileSystem fs = cluster.getFileSystem();
assertTrue("Not a HDFS: " + fs.getUri(),
- fs instanceof DistributedFileSystem);
+ fs instanceof DistributedFileSystem);
try {
fs.mkdirs(new Path(new Path("parent"), "child"));
try {
@@ -224,6 +229,7 @@ public class TestDFSShell {
shell.setConf(conf);
try {
+ cluster.waitActive();
Path myPath = new Path("/test/dir");
assertTrue(fs.mkdirs(myPath));
assertTrue(fs.exists(myPath));
@@ -251,7 +257,7 @@ public class TestDFSShell {
assertTrue(val == 0);
String returnString = out.toString();
out.reset();
- // Check if size matchs as expected
+ // Check if size matches as expected
assertThat(returnString, containsString(myFileLength.toString()));
assertThat(returnString, containsString(myFileDiskUsed.toString()));
assertThat(returnString, containsString(myFile2Length.toString()));
@@ -308,7 +314,232 @@ public class TestDFSShell {
System.setOut(psBackup);
cluster.shutdown();
}
+ }
+
+ @Test (timeout = 180000)
+ public void testDuSnapshots() throws IOException {
+ final int replication = 2;
+ final Configuration conf = new HdfsConfiguration();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(replication).build();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final PrintStream psBackup = System.out;
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ final PrintStream psOut = new PrintStream(out);
+ final FsShell shell = new FsShell();
+ shell.setConf(conf);
+
+ try {
+ System.setOut(psOut);
+ cluster.waitActive();
+ final Path parent = new Path("/test");
+ final Path dir = new Path(parent, "dir");
+ mkdir(dfs, dir);
+ final Path file = new Path(dir, "file");
+ writeFile(dfs, file);
+ final Path file2 = new Path(dir, "file2");
+ writeFile(dfs, file2);
+ final Long fileLength = dfs.getFileStatus(file).getLen();
+ final Long fileDiskUsed = fileLength * replication;
+ final Long file2Length = dfs.getFileStatus(file2).getLen();
+ final Long file2DiskUsed = file2Length * replication;
+
+ /*
+ * Construct dir as follows:
+ * /test/dir/file <- this will later be deleted after snapshot taken.
+ * /test/dir/newfile <- this will be created after snapshot taken.
+ * /test/dir/file2
+ * Snapshot enabled on /test
+ */
+
+ // test -du on /test/dir
+ int ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", dir.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ String returnString = out.toString();
+ LOG.info("-du return is:\n" + returnString);
+ // Check if size matches as expected
+ assertTrue(returnString.contains(fileLength.toString()));
+ assertTrue(returnString.contains(fileDiskUsed.toString()));
+ assertTrue(returnString.contains(file2Length.toString()));
+ assertTrue(returnString.contains(file2DiskUsed.toString()));
+ out.reset();
+
+ // take a snapshot, then remove file and add newFile
+ final String snapshotName = "ss1";
+ final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName);
+ dfs.allowSnapshot(parent);
+ assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath));
+ rmr(dfs, file);
+ final Path newFile = new Path(dir, "newfile");
+ writeFile(dfs, newFile);
+ final Long newFileLength = dfs.getFileStatus(newFile).getLen();
+ final Long newFileDiskUsed = newFileLength * replication;
+
+ // test -du -s on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", "-s", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du -s return is:\n" + returnString);
+ Long combinedLength = fileLength + file2Length + newFileLength;
+ Long combinedDiskUsed = fileDiskUsed + file2DiskUsed + newFileDiskUsed;
+ assertTrue(returnString.contains(combinedLength.toString()));
+ assertTrue(returnString.contains(combinedDiskUsed.toString()));
+ out.reset();
+
+ // test -du on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du return is:\n" + returnString);
+ assertTrue(returnString.contains(combinedLength.toString()));
+ assertTrue(returnString.contains(combinedDiskUsed.toString()));
+ out.reset();
+
+ // test -du -s -x on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", "-s", "-x", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du -s -x return is:\n" + returnString);
+ Long exludeSnapshotLength = file2Length + newFileLength;
+ Long excludeSnapshotDiskUsed = file2DiskUsed + newFileDiskUsed;
+ assertTrue(returnString.contains(exludeSnapshotLength.toString()));
+ assertTrue(returnString.contains(excludeSnapshotDiskUsed.toString()));
+ out.reset();
+ // test -du -x on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", "-x", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du -x return is:\n" + returnString);
+ assertTrue(returnString.contains(exludeSnapshotLength.toString()));
+ assertTrue(returnString.contains(excludeSnapshotDiskUsed.toString()));
+ out.reset();
+ } finally {
+ System.setOut(psBackup);
+ cluster.shutdown();
+ }
+ }
+
+ @Test (timeout = 180000)
+ public void testCountSnapshots() throws IOException {
+ final int replication = 2;
+ final Configuration conf = new HdfsConfiguration();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(replication).build();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final PrintStream psBackup = System.out;
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ final PrintStream psOut = new PrintStream(out);
+ System.setOut(psOut);
+ final FsShell shell = new FsShell();
+ shell.setConf(conf);
+
+ try {
+ cluster.waitActive();
+ final Path parent = new Path("/test");
+ final Path dir = new Path(parent, "dir");
+ mkdir(dfs, dir);
+ final Path file = new Path(dir, "file");
+ writeFile(dfs, file);
+ final Path file2 = new Path(dir, "file2");
+ writeFile(dfs, file2);
+ final long fileLength = dfs.getFileStatus(file).getLen();
+ final long file2Length = dfs.getFileStatus(file2).getLen();
+ final Path dir2 = new Path(parent, "dir2");
+ mkdir(dfs, dir2);
+
+ /*
+ * Construct dir as follows:
+ * /test/dir/file <- this will later be deleted after snapshot taken.
+ * /test/dir/newfile <- this will be created after snapshot taken.
+ * /test/dir/file2
+ * /test/dir2 <- this will later be deleted after snapshot taken.
+ * Snapshot enabled on /test
+ */
+
+ // take a snapshot
+ // then create /test/dir/newfile and remove /test/dir/file, /test/dir2
+ final String snapshotName = "s1";
+ final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName);
+ dfs.allowSnapshot(parent);
+ assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath));
+ rmr(dfs, file);
+ rmr(dfs, dir2);
+ final Path newFile = new Path(dir, "new file");
+ writeFile(dfs, newFile);
+ final Long newFileLength = dfs.getFileStatus(newFile).getLen();
+
+ // test -count on /test. Include header for easier debugging.
+ int val = -1;
+ try {
+ val = shell.run(new String[] {"-count", "-v", parent.toString() });
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, val);
+ String returnString = out.toString();
+ LOG.info("-count return is:\n" + returnString);
+ Scanner in = new Scanner(returnString);
+ in.nextLine();
+ assertEquals(3, in.nextLong()); //DIR_COUNT
+ assertEquals(3, in.nextLong()); //FILE_COUNT
+ assertEquals(fileLength + file2Length + newFileLength,
+ in.nextLong()); //CONTENT_SIZE
+ out.reset();
+
+ // test -count -x on /test. Include header for easier debugging.
+ val = -1;
+ try {
+ val =
+ shell.run(new String[] {"-count", "-x", "-v", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, val);
+ returnString = out.toString();
+ LOG.info("-count -x return is:\n" + returnString);
+ in = new Scanner(returnString);
+ in.nextLine();
+ assertEquals(2, in.nextLong()); //DIR_COUNT
+ assertEquals(2, in.nextLong()); //FILE_COUNT
+ assertEquals(file2Length + newFileLength, in.nextLong()); //CONTENT_SIZE
+ out.reset();
+ } finally {
+ System.setOut(psBackup);
+ cluster.shutdown();
+ }
}
@Test (timeout = 30000)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org