You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2017/07/07 21:18:16 UTC
[02/50] [abbrv] hadoop git commit: Add -E option in 'ls' to list
erasure coding policy of each file and directory if applicable. Contributed
by luhuichun via lei.
Add -E option in 'ls' to list erasure coding policy of each file and directory if applicable. Contributed by luhuichun via lei.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6df0fdb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6df0fdb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6df0fdb
Branch: refs/heads/YARN-5734
Commit: d6df0fdbbda42b4ddab3810b5ac57336c6241ba7
Parents: 25d891a
Author: Lei Xu <le...@apache.org>
Authored: Wed Jun 28 13:47:23 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Wed Jun 28 13:47:23 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/shell/Ls.java | 63 +++++++++++++++-----
.../src/site/markdown/FileSystemShell.md | 4 +-
.../src/test/resources/testConf.xml | 6 +-
.../test/resources/testErasureCodingConf.xml | 34 +++++++++++
4 files changed, 89 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6df0fdb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index 47e87f5..221b3cb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.ContentSummary;
/**
* Get a listing of all files in that match the file patterns.
@@ -54,13 +55,14 @@ class Ls extends FsCommand {
private static final String OPTION_MTIME = "t";
private static final String OPTION_ATIME = "u";
private static final String OPTION_SIZE = "S";
+ private static final String OPTION_ECPOLICY = "e";
public static final String NAME = "ls";
public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-" +
OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" +
OPTION_HIDENONPRINTABLE + "] [-" + OPTION_RECURSIVE + "] [-" +
OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] [-" +
- OPTION_ATIME + "] [<path> ...]";
+ OPTION_ATIME + "] [-" + OPTION_ECPOLICY +"] [<path> ...]";
public static final String DESCRIPTION =
"List the contents that match the specified file pattern. If " +
@@ -91,7 +93,9 @@ class Ls extends FsCommand {
" Reverse the order of the sort.\n" +
" -" + OPTION_ATIME +
" Use time of last access instead of modification for\n" +
- " display and sorting.";
+ " display and sorting.\n"+
+ " -" + OPTION_ECPOLICY +
+ " Display the erasure coding policy of files and directories.\n";
protected final SimpleDateFormat dateFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm");
@@ -104,6 +108,7 @@ class Ls extends FsCommand {
private boolean orderTime;
private boolean orderSize;
private boolean useAtime;
+ private boolean displayECPolicy;
private Comparator<PathData> orderComparator;
protected boolean humanReadable = false;
@@ -129,7 +134,7 @@ class Ls extends FsCommand {
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE,
OPTION_PATHONLY, OPTION_DIRECTORY, OPTION_HUMAN,
OPTION_HIDENONPRINTABLE, OPTION_RECURSIVE, OPTION_REVERSE,
- OPTION_MTIME, OPTION_SIZE, OPTION_ATIME);
+ OPTION_MTIME, OPTION_SIZE, OPTION_ATIME, OPTION_ECPOLICY);
cf.parse(args);
pathOnly = cf.getOpt(OPTION_PATHONLY);
dirRecurse = !cf.getOpt(OPTION_DIRECTORY);
@@ -140,6 +145,7 @@ class Ls extends FsCommand {
orderTime = cf.getOpt(OPTION_MTIME);
orderSize = !orderTime && cf.getOpt(OPTION_SIZE);
useAtime = cf.getOpt(OPTION_ATIME);
+ displayECPolicy = cf.getOpt(OPTION_ECPOLICY);
if (args.isEmpty()) args.add(Path.CUR_DIR);
initialiseOrderComparator();
@@ -245,25 +251,42 @@ class Ls extends FsCommand {
return;
}
FileStatus stat = item.stat;
- String line = String.format(lineFormat,
- (stat.isDirectory() ? "d" : "-"),
- stat.getPermission() + (stat.getPermission().getAclBit() ? "+" : " "),
- (stat.isFile() ? stat.getReplication() : "-"),
- stat.getOwner(),
- stat.getGroup(),
- formatSize(stat.getLen()),
- dateFormat.format(new Date(isUseAtime()
- ? stat.getAccessTime()
- : stat.getModificationTime())),
- isHideNonPrintable() ? new PrintableString(item.toString()) : item);
- out.println(line);
+ if (displayECPolicy) {
+ ContentSummary contentSummary = item.fs.getContentSummary(item.path);
+ String line = String.format(lineFormat,
+ (stat.isDirectory() ? "d" : "-"),
+ stat.getPermission() + (stat.getPermission().getAclBit() ? "+" : " "),
+ (stat.isFile() ? stat.getReplication() : "-"),
+ stat.getOwner(),
+ stat.getGroup(),
+ contentSummary.getErasureCodingPolicy(),
+ formatSize(stat.getLen()),
+ dateFormat.format(new Date(isUseAtime()
+ ? stat.getAccessTime()
+ : stat.getModificationTime())),
+ isHideNonPrintable() ? new PrintableString(item.toString()) : item);
+ out.println(line);
+ } else {
+ String line = String.format(lineFormat,
+ (stat.isDirectory() ? "d" : "-"),
+ stat.getPermission() + (stat.getPermission().getAclBit() ? "+" : " "),
+ (stat.isFile() ? stat.getReplication() : "-"),
+ stat.getOwner(),
+ stat.getGroup(),
+ formatSize(stat.getLen()),
+ dateFormat.format(new Date(isUseAtime()
+ ? stat.getAccessTime()
+ : stat.getModificationTime())),
+ isHideNonPrintable() ? new PrintableString(item.toString()) : item);
+ out.println(line);
+ }
}
/**
* Compute column widths and rebuild the format string
* @param items to find the max field width for each column
*/
- private void adjustColumnWidths(PathData items[]) {
+ private void adjustColumnWidths(PathData items[]) throws IOException {
for (PathData item : items) {
FileStatus stat = item.stat;
maxRepl = maxLength(maxRepl, stat.getReplication());
@@ -278,6 +301,14 @@ class Ls extends FsCommand {
// Do not use '%-0s' as a formatting conversion, since it will throw a
// a MissingFormatWidthException if it is used in String.format().
// http://docs.oracle.com/javase/1.5.0/docs/api/java/util/Formatter.html#intFlags
+ if(displayECPolicy){
+ int maxEC=0;
+ for (PathData item : items) {
+ ContentSummary contentSummary = item.fs.getContentSummary(item.path);
+ maxEC=maxLength(maxEC,contentSummary.getErasureCodingPolicy().length());
+ }
+ fmt.append(" %"+maxEC+"s ");
+ }
fmt.append((maxOwner > 0) ? "%-" + maxOwner + "s " : "%s");
fmt.append((maxGroup > 0) ? "%-" + maxGroup + "s " : "%s");
fmt.append("%" + maxLen + "s ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6df0fdb/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 7786642..0a594ab 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -410,7 +410,7 @@ Return usage output.
ls
----
-Usage: `hadoop fs -ls [-C] [-d] [-h] [-q] [-R] [-t] [-S] [-r] [-u] <args> `
+Usage: `hadoop fs -ls [-C] [-d] [-h] [-q] [-R] [-t] [-S] [-r] [-u] [-e] <args> `
Options:
@@ -423,6 +423,7 @@ Options:
* -S: Sort output by file size.
* -r: Reverse the sort order.
* -u: Use access time rather than modification time for display and sorting.
+* -e: Display the erasure coding policy of files and directories only.
For a file ls returns stat on the file with the following format:
@@ -437,6 +438,7 @@ Files within a directory are order by filename by default.
Example:
* `hadoop fs -ls /user/hadoop/file1`
+* `hadoop fs -ls -e /ecdir`
Exit Code:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6df0fdb/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 710f063..342b17c 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -54,7 +54,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-ls \[-C\] \[-d\] \[-h\] \[-q\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)*</expected-output>
+ <expected-output>^-ls \[-C\] \[-d\] \[-h\] \[-q\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[-e\] \[<path> \.\.\.\] :( |\t)*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -136,6 +136,10 @@
<type>RegexpComparator</type>
<expected-output>^( |\t)*display and sorting\.</expected-output>
</comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*-e\s+Display the erasure coding policy of files and directories\.</expected-output>
+ </comparator>
</comparators>
</test>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6df0fdb/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index 791e685..127effc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -773,5 +773,39 @@
</comparator>
</comparators>
</test>
+
+ <test> <!-- TESTED -->
+ <description>ls: file using absolute path and option -e to show erasure coding policy of a directory</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /ecdir</command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-6-3-64k</ec-admin-command>
+ <command>-fs NAMENODE -touchz /ecdir/file1</command>
+ <command>-fs NAMENODE -touchz /ecdir/file2</command>
+ <command>-fs NAMENODE -touchz /ecdir/file3</command>
+ <command>-fs NAMENODE -ls -e /ecdir</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /ecdir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>Found [0-9] items</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^-rw-r--r--( )*1( )*USERNAME( )*supergroup( )*[A-Za-z0-9-]{1,}( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/ecdir/file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^-rw-r--r--( )*1( )*USERNAME( )*supergroup( )*[A-Za-z0-9-]{1,}( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/ecdir/file2</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^-rw-r--r--( )*1( )*USERNAME( )*supergroup( )*[A-Za-z0-9-]{1,}( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/ecdir/file3</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
</tests>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org