You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2009/05/27 12:49:57 UTC
svn commit: r779102 - in /hadoop/core/trunk: ./
src/core/org/apache/hadoop/fs/ src/core/org/apache/hadoop/util/
src/test/hdfs/org/apache/hadoop/cli/
src/test/hdfs/org/apache/hadoop/cli/clitest_data/
Author: tomwhite
Date: Wed May 27 10:49:56 2009
New Revision: 779102
URL: http://svn.apache.org/viewvc?rev=779102&view=rev
Log:
HADOOP-4861. Add disk usage with human-readable size (-duh). Contributed by Todd Lipcon.
Added:
hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/cli/clitest_data/data1k
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/build.xml
hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java
hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java
hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/cli/testHDFSConf.xml
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=779102&r1=779101&r2=779102&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed May 27 10:49:56 2009
@@ -128,6 +128,9 @@
HADOOP-5815. Sqoop: A database import tool for Hadoop.
(Aaron Kimball via tomwhite)
+ HADOOP-4861. Add disk usage with human-readable size (-duh).
+ (Todd Lipcon via tomwhite)
+
IMPROVEMENTS
HADOOP-4565. Added CombineFileInputFormat to use data locality information
Modified: hadoop/core/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/build.xml?rev=779102&r1=779101&r2=779102&view=diff
==============================================================================
--- hadoop/core/trunk/build.xml (original)
+++ hadoop/core/trunk/build.xml Wed May 27 10:49:56 2009
@@ -709,6 +709,7 @@
<copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data30bytes" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data60bytes" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data120bytes" todir="${test.cache.data}"/>
+ <copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data1k" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19" todir="${test.cache.data}"/>
</target>
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java?rev=779102&r1=779101&r2=779102&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java Wed May 27 10:49:56 2009
@@ -61,6 +61,7 @@
static final String COPYTOLOCAL_SHORT_USAGE = GET_SHORT_USAGE.replace(
"-get", "-copyToLocal");
static final String TAIL_USAGE="-tail [-f] <file>";
+ static final String DU_USAGE="-du [-s] [-h] <paths...>";
/**
*/
@@ -670,58 +671,98 @@
/**
* Show the size of all files that match the file pattern <i>src</i>
- * @param src a file pattern specifying source files
+ * @param cmd
+ * @param pos ignore anything before this pos in cmd
* @throws IOException
* @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
*/
- void du(String src) throws IOException {
- Path srcPath = new Path(src);
- FileSystem srcFs = srcPath.getFileSystem(getConf());
- Path[] pathItems = FileUtil.stat2Paths(srcFs.globStatus(srcPath),
- srcPath);
- FileStatus items[] = srcFs.listStatus(pathItems);
- if ((items == null) || ((items.length == 0) &&
- (!srcFs.exists(srcPath)))){
- throw new FileNotFoundException("Cannot access " + src
- + ": No such file or directory.");
- } else {
- System.out.println("Found " + items.length + " items");
- int maxLength = 10;
-
- long length[] = new long[items.length];
- for (int i = 0; i < items.length; i++) {
- length[i] = items[i].isDir() ?
- srcFs.getContentSummary(items[i].getPath()).getLength() :
- items[i].getLen();
- int len = String.valueOf(length[i]).length();
- if (len > maxLength) maxLength = len;
+ void du(String[] cmd, int pos) throws IOException {
+ CommandFormat c = new CommandFormat(
+ "du", 0, Integer.MAX_VALUE, "h", "s");
+ List<String> params;
+ try {
+ params = c.parse(cmd, pos);
+ } catch (IllegalArgumentException iae) {
+ System.err.println("Usage: java FsShell " + DU_USAGE);
+ throw iae;
+ }
+ boolean humanReadable = c.getOpt("h");
+ boolean summary = c.getOpt("s");
+
+ // Default to cwd
+ if (params.isEmpty()) {
+ params.add(".");
+ }
+
+ List<UsagePair> usages = new ArrayList<UsagePair>();
+
+ for (String src : params) {
+ Path srcPath = new Path(src);
+ FileSystem srcFs = srcPath.getFileSystem(getConf());
+ FileStatus globStatus[] = srcFs.globStatus(srcPath);
+ FileStatus statusToPrint[];
+
+ if (summary) {
+ statusToPrint = globStatus;
+ } else {
+ Path statPaths[] = FileUtil.stat2Paths(globStatus, srcPath);
+ statusToPrint = srcFs.listStatus(statPaths);
}
- for(int i = 0; i < items.length; i++) {
- System.out.printf("%-"+ (maxLength + BORDER) +"d", length[i]);
- System.out.println(items[i].getPath());
+ if ((statusToPrint == null) || ((statusToPrint.length == 0) &&
+ (!srcFs.exists(srcPath)))){
+ throw new FileNotFoundException("Cannot access " + src
+ + ": No such file or directory.");
+ }
+
+ if (!summary) {
+ System.out.println("Found " + statusToPrint.length + " items");
+ }
+
+ for (FileStatus stat : statusToPrint) {
+ long length;
+ if (summary || stat.isDir()) {
+ length = srcFs.getContentSummary(stat.getPath()).getLength();
+ } else {
+ length = stat.getLen();
+ }
+
+ usages.add(new UsagePair(String.valueOf(stat.getPath()), length));
}
}
+ printUsageSummary(usages, humanReadable);
}
/**
* Show the summary disk usage of each dir/file
* that matches the file pattern <i>src</i>
- * @param src a file pattern specifying source files
+ * @param cmd
+ * @param pos ignore anything before this pos in cmd
* @throws IOException
* @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
*/
- void dus(String src) throws IOException {
- Path srcPath = new Path(src);
- FileSystem srcFs = srcPath.getFileSystem(getConf());
- FileStatus status[] = srcFs.globStatus(new Path(src));
- if (status==null || status.length==0) {
- throw new FileNotFoundException("Cannot access " + src +
- ": No such file or directory.");
+ void dus(String[] cmd, int pos) throws IOException {
+ String newcmd[] = new String[cmd.length + 1];
+ System.arraycopy(cmd, 0, newcmd, 0, cmd.length);
+ newcmd[cmd.length] = "-s";
+ du(newcmd, pos);
+ }
+
+ private void printUsageSummary(List<UsagePair> usages,
+ boolean humanReadable) {
+ int maxColumnWidth = 0;
+ for (UsagePair usage : usages) {
+ String toPrint = humanReadable ?
+ StringUtils.humanReadableInt(usage.bytes) : String.valueOf(usage.bytes);
+ if (toPrint.length() > maxColumnWidth) {
+ maxColumnWidth = toPrint.length();
+ }
}
- for(int i=0; i<status.length; i++) {
- long totalSize = srcFs.getContentSummary(status[i].getPath()).getLength();
- String pathStr = status[i].getPath().toString();
- System.out.println(("".equals(pathStr)?".":pathStr) + "\t" + totalSize);
+
+ for (UsagePair usage : usages) {
+ String toPrint = humanReadable ?
+ StringUtils.humanReadableInt(usage.bytes) : String.valueOf(usage.bytes);
+ System.out.printf("%-"+ (maxColumnWidth + BORDER) +"s", toPrint);
+ System.out.println(usage.path);
}
}
@@ -1558,10 +1599,6 @@
delete(argv[i], true);
} else if ("-df".equals(cmd)) {
df(argv[i]);
- } else if ("-du".equals(cmd)) {
- du(argv[i]);
- } else if ("-dus".equals(cmd)) {
- dus(argv[i]);
} else if (Count.matches(cmd)) {
new Count(argv, i, getConf()).runAll();
} else if ("-ls".equals(cmd)) {
@@ -1809,17 +1846,9 @@
df(null);
}
} else if ("-du".equals(cmd)) {
- if (i < argv.length) {
- exitCode = doall(cmd, argv, i);
- } else {
- du(".");
- }
+ du(argv, i);
} else if ("-dus".equals(cmd)) {
- if (i < argv.length) {
- exitCode = doall(cmd, argv, i);
- } else {
- dus(".");
- }
+ dus(argv, i);
} else if (Count.matches(cmd)) {
exitCode = new Count(argv, i, getConf()).runAll();
} else if ("-mkdir".equals(cmd)) {
@@ -1922,4 +1951,18 @@
throw new IOException("Multiple IOExceptions: " + exceptions);
}
}
+
+
+ /**
+ * Utility class for a line of du output
+ */
+ private static class UsagePair {
+ public String path;
+ public long bytes;
+
+ public UsagePair(String path, long bytes) {
+ this.path = path;
+ this.bytes = bytes;
+ }
+ }
}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java?rev=779102&r1=779101&r2=779102&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java Wed May 27 10:49:56 2009
@@ -88,7 +88,8 @@
double result = number;
String suffix = "";
if (absNumber < 1024) {
- // nothing
+ // since no division has occurred, don't format with a decimal point
+ return String.valueOf(number);
} else if (absNumber < 1024 * 1024) {
result = number / 1024.0;
suffix = "k";
Added: hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/cli/clitest_data/data1k
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/cli/clitest_data/data1k?rev=779102&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/cli/clitest_data/data1k (added)
+++ hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/cli/clitest_data/data1k Wed May 27 10:49:56 2009
@@ -0,0 +1,71 @@
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
Modified: hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/cli/testHDFSConf.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/cli/testHDFSConf.xml?rev=779102&r1=779101&r2=779102&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/cli/testHDFSConf.xml (original)
+++ hadoop/core/trunk/src/test/hdfs/org/apache/hadoop/cli/testHDFSConf.xml Wed May 27 10:49:56 2009
@@ -1132,6 +1132,33 @@
</comparator>
</comparators>
</test>
+
+ <test> <!-- TESTED -->
+ <description>duh: Test for hdfs:// path - directory</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir hdfs:///dir0</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data15bytes hdfs:///dir0/data15bytes</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data1k hdfs:///dir0/data1k</command>
+ <command>-fs NAMENODE -du -h hdfs:///dir0</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmr hdfs:///dir0</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>TokenComparator</type>
+ <expected-output>Found 2 items</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^15( |\t)*hdfs://localhost[.a-z]*:[0-9]*/dir0/data15bytes</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^1.0k( |\t)*hdfs://localhost[.a-z]*:[0-9]*/dir0/data1k</expected-output>
+ </comparator>
+ </comparators>
+ </test>
<test> <!-- TESTED -->
<description>du: Test for hdfs:// path - directory using globbing</description>
@@ -1314,7 +1341,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+ <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
</comparator>
</comparators>
</test>
@@ -1346,7 +1373,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0( |\t)*450</expected-output>
+ <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0</expected-output>
</comparator>
</comparators>
</test>
@@ -1384,7 +1411,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+ <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
</comparator>
</comparators>
</test>
@@ -1417,7 +1444,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+ <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
</comparator>
</comparators>
</test>
@@ -1455,7 +1482,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+ <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
</comparator>
</comparators>
</test>
@@ -1487,7 +1514,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+ <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
</comparator>
</comparators>
</test>
@@ -1525,7 +1552,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+ <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
</comparator>
</comparators>
</test>
@@ -5485,7 +5512,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
</comparator>
</comparators>
</test>
@@ -5502,7 +5529,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0</expected-output>
</comparator>
</comparators>
</test>
@@ -5522,19 +5549,19 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir1(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir1</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir2(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir2</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir3(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir3</expected-output>
</comparator>
</comparators>
</test>
@@ -5554,19 +5581,19 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir1(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir1</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir2(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir2</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir3(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir3</expected-output>
</comparator>
</comparators>
</test>
@@ -5617,7 +5644,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*</expected-output>
</comparator>
</comparators>
</test>
@@ -5634,19 +5661,19 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir1(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir1</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir2(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir2</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir3(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir3</expected-output>
</comparator>
</comparators>
</test>
@@ -5697,7 +5724,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
</comparator>
</comparators>
</test>
@@ -5714,19 +5741,19 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir1(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir1</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir2(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir2</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir3(|\t)*0</expected-output>
+ <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir3</expected-output>
</comparator>
</comparators>
</test>