You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aa...@apache.org on 2015/05/19 10:59:59 UTC
[3/3] hadoop git commit: HADOOP-10971. Add -C flag to make `hadoop fs
-ls` print filenames only. Contributed by Kengo Seki.
HADOOP-10971. Add -C flag to make `hadoop fs -ls` print filenames only. Contributed by Kengo Seki.
(cherry picked from commit 3b50dcdce4ffe3d4e5892fca84909ff22be28739)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb49967e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb49967e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb49967e
Branch: refs/heads/branch-2
Commit: fb49967e97afa771fe5bd95085d5d92ea2a8a484
Parents: df6d242
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue May 19 17:25:27 2015 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue May 19 17:58:52 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../java/org/apache/hadoop/fs/shell/Ls.java | 36 +++++++++--
.../src/site/markdown/FileSystemShell.md | 3 +-
.../java/org/apache/hadoop/fs/shell/TestLs.java | 67 ++++++++++++++++++++
.../src/test/resources/testConf.xml | 6 +-
5 files changed, 106 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb49967e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 93304dc..4057c4e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -22,6 +22,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-8934. Shell command ls should include sort options (Jonathan Allen
via aw)
+ HADOOP-10971. Add -C flag to make `hadoop fs -ls` print filenames only.
+ (Kengo Seki via aajisaka)
+
IMPROVEMENTS
HADOOP-6842. "hadoop fs -text" does not give a useful text representation
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb49967e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index 0e46700..171d221 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -43,6 +43,7 @@ class Ls extends FsCommand {
factory.addClass(Lsr.class, "-lsr");
}
+ private static final String OPTION_PATHONLY = "C";
private static final String OPTION_DIRECTORY = "d";
private static final String OPTION_HUMAN = "h";
private static final String OPTION_RECURSIVE = "R";
@@ -52,10 +53,10 @@ class Ls extends FsCommand {
private static final String OPTION_SIZE = "S";
public static final String NAME = "ls";
- public static final String USAGE = "[-" + OPTION_DIRECTORY + "] [-"
- + OPTION_HUMAN + "] " + "[-" + OPTION_RECURSIVE + "] [-" + OPTION_MTIME
- + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] " + "[-"
- + OPTION_ATIME + "] [<path> ...]";
+ public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-"
+ + OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" + OPTION_RECURSIVE
+ + "] [-" + OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE
+ + "] [-" + OPTION_ATIME + "] [<path> ...]";
public static final String DESCRIPTION =
"List the contents that match the specified file pattern. If " +
@@ -67,6 +68,8 @@ class Ls extends FsCommand {
"\tpermissions - userId groupId sizeOfDirectory(in bytes) modificationDate(yyyy-MM-dd HH:mm) directoryName\n\n" +
"and file entries are of the form:\n" +
"\tpermissions numberOfReplicas userId groupId sizeOfFile(in bytes) modificationDate(yyyy-MM-dd HH:mm) fileName\n\n" +
+ " -" + OPTION_PATHONLY +
+ " Display the paths of files and directories only.\n" +
" -" + OPTION_DIRECTORY +
" Directories are listed as plain files.\n" +
" -" + OPTION_HUMAN +
@@ -89,6 +92,7 @@ class Ls extends FsCommand {
protected int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0;
protected String lineFormat;
+ private boolean pathOnly;
protected boolean dirRecurse;
private boolean orderReverse;
private boolean orderTime;
@@ -107,10 +111,11 @@ class Ls extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args)
throws IOException {
- CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE,
+ CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, OPTION_PATHONLY,
OPTION_DIRECTORY, OPTION_HUMAN, OPTION_RECURSIVE, OPTION_REVERSE,
OPTION_MTIME, OPTION_SIZE, OPTION_ATIME);
cf.parse(args);
+ pathOnly = cf.getOpt(OPTION_PATHONLY);
dirRecurse = !cf.getOpt(OPTION_DIRECTORY);
setRecursive(cf.getOpt(OPTION_RECURSIVE) && dirRecurse);
humanReadable = cf.getOpt(OPTION_HUMAN);
@@ -124,6 +129,15 @@ class Ls extends FsCommand {
}
/**
+ * Should display only paths of files and directories.
+ * @return true display paths only, false display all fields
+ */
+ @InterfaceAudience.Private
+ boolean isPathOnly() {
+ return this.pathOnly;
+ }
+
+ /**
* Should the contents of the directory be shown or just the directory?
* @return true if directory contents, false if just directory
*/
@@ -191,15 +205,23 @@ class Ls extends FsCommand {
protected void processPaths(PathData parent, PathData ... items)
throws IOException {
if (parent != null && !isRecursive() && items.length != 0) {
- out.println("Found " + items.length + " items");
+ if (!pathOnly) {
+ out.println("Found " + items.length + " items");
+ }
Arrays.sort(items, getOrderComparator());
}
- adjustColumnWidths(items);
+ if (!pathOnly) {
+ adjustColumnWidths(items);
+ }
super.processPaths(parent, items);
}
@Override
protected void processPath(PathData item) throws IOException {
+ if (pathOnly) {
+ out.println(item.toString());
+ return;
+ }
FileStatus stat = item.stat;
String line = String.format(lineFormat,
(stat.isDirectory() ? "d" : "-"),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb49967e/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index aaf7ef4..ae2b0ef 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -382,10 +382,11 @@ Return usage output.
ls
----
-Usage: `hadoop fs -ls [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
+Usage: `hadoop fs -ls [-C] [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
Options:
+* -C: Display the paths of files and directories only.
* -d: Directories are listed as plain files.
* -h: Format file sizes in a human-readable fashion (eg 64.0m instead of 67108864).
* -R: Recursively list subdirectories encountered.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb49967e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
index 66403db..4a9103f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
@@ -74,6 +74,24 @@ public class TestLs {
LinkedList<String> options = new LinkedList<String>();
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
+ assertTrue(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertFalse(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertFalse(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ }
+
+ // check the -C option is recognised
+ @Test
+ public void processOptionsPathOnly() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-C");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertTrue(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -90,6 +108,7 @@ public class TestLs {
options.add("-d");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertFalse(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -106,6 +125,7 @@ public class TestLs {
options.add("-h");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertTrue(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -122,6 +142,7 @@ public class TestLs {
options.add("-R");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertTrue(ls.isRecursive());
@@ -138,6 +159,7 @@ public class TestLs {
options.add("-r");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -154,6 +176,7 @@ public class TestLs {
options.add("-S");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -170,6 +193,7 @@ public class TestLs {
options.add("-t");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -187,6 +211,7 @@ public class TestLs {
options.add("-S");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -205,6 +230,7 @@ public class TestLs {
options.add("-r");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -221,6 +247,7 @@ public class TestLs {
options.add("-u");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -234,6 +261,7 @@ public class TestLs {
@Test
public void processOptionsAll() throws IOException {
LinkedList<String> options = new LinkedList<String>();
+ options.add("-C"); // show file path only
options.add("-d"); // directory
options.add("-h"); // human readable
options.add("-R"); // recursive
@@ -243,6 +271,7 @@ public class TestLs {
options.add("-u"); // show atime
Ls ls = new Ls();
ls.processOptions(options);
+ assertTrue(ls.isPathOnly());
assertFalse(ls.isDirRecurse());
assertTrue(ls.isHumanReadable());
assertFalse(ls.isRecursive()); // -d overrules -R
@@ -981,6 +1010,44 @@ public class TestLs {
verifyNoMoreInteractions(out);
}
+ // check path only display (-C option)
+ @Test
+ public void processPathDirectoryPathOnly() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-C");
+ ls.processOptions(options);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println(testfile01.getPath().toString());
+ inOrder.verify(out).println(testfile02.getPath().toString());
+ inOrder.verify(out).println(testfile03.getPath().toString());
+ inOrder.verify(out).println(testfile04.getPath().toString());
+ inOrder.verify(out).println(testfile05.getPath().toString());
+ inOrder.verify(out).println(testfile06.getPath().toString());
+ verifyNoMoreInteractions(out);
+ }
+
// check the deprecated flag isn't set
@Test
public void isDeprecated() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb49967e/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index ae5f44c..8f48ccc 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -54,7 +54,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-ls \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)*</expected-output>
+ <expected-output>^-ls \[-C\] \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -94,6 +94,10 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
+ <expected-output>^\s*-C\s+Display the paths of files and directories only\.( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
<expected-output>^\s*-d\s+Directories are listed as plain files\.( )*</expected-output>
</comparator>
<comparator>