You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2016/06/13 18:45:45 UTC
hadoop git commit: HADOOP-13079. Add -q option to Ls to print ?
instead of non-printable characters. Contributed by John Zhuge.
Repository: hadoop
Updated Branches:
refs/heads/trunk 28b66ae91 -> 0accc3306
HADOOP-13079. Add -q option to Ls to print ? instead of non-printable characters. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0accc330
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0accc330
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0accc330
Branch: refs/heads/trunk
Commit: 0accc3306d830c3f2b16c4b8abf68729c7aba6cb
Parents: 28b66ae
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Jun 13 11:43:46 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Jun 13 11:43:46 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/shell/Ls.java | 28 +++++--
.../apache/hadoop/fs/shell/PrintableString.java | 72 ++++++++++++++++
.../src/site/markdown/FileSystemShell.md | 3 +-
.../org/apache/hadoop/fs/TestFsShellList.java | 78 ++++++++++++++++++
.../hadoop/fs/shell/TestPrintableString.java | 87 ++++++++++++++++++++
.../apache/hadoop/fs/shell/package-info.java | 26 ++++++
.../src/test/resources/testConf.xml | 6 +-
7 files changed, 291 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0accc330/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index 67348c6..47e87f5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -48,6 +48,7 @@ class Ls extends FsCommand {
private static final String OPTION_PATHONLY = "C";
private static final String OPTION_DIRECTORY = "d";
private static final String OPTION_HUMAN = "h";
+ private static final String OPTION_HIDENONPRINTABLE = "q";
private static final String OPTION_RECURSIVE = "R";
private static final String OPTION_REVERSE = "r";
private static final String OPTION_MTIME = "t";
@@ -55,10 +56,11 @@ class Ls extends FsCommand {
private static final String OPTION_SIZE = "S";
public static final String NAME = "ls";
- public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-"
- + OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" + OPTION_RECURSIVE
- + "] [-" + OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE
- + "] [-" + OPTION_ATIME + "] [<path> ...]";
+ public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-" +
+ OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" +
+ OPTION_HIDENONPRINTABLE + "] [-" + OPTION_RECURSIVE + "] [-" +
+ OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] [-" +
+ OPTION_ATIME + "] [<path> ...]";
public static final String DESCRIPTION =
"List the contents that match the specified file pattern. If " +
@@ -77,6 +79,8 @@ class Ls extends FsCommand {
" -" + OPTION_HUMAN +
" Formats the sizes of files in a human-readable fashion\n" +
" rather than a number of bytes.\n" +
+ " -" + OPTION_HIDENONPRINTABLE +
+ " Print ? instead of non-printable characters.\n" +
" -" + OPTION_RECURSIVE +
" Recursively list the contents of directories.\n" +
" -" + OPTION_MTIME +
@@ -104,6 +108,9 @@ class Ls extends FsCommand {
protected boolean humanReadable = false;
+ /** Whether to print ? instead of non-printable characters. */
+ private boolean hideNonPrintable = false;
+
protected Ls() {}
protected Ls(Configuration conf) {
@@ -119,14 +126,16 @@ class Ls extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args)
throws IOException {
- CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, OPTION_PATHONLY,
- OPTION_DIRECTORY, OPTION_HUMAN, OPTION_RECURSIVE, OPTION_REVERSE,
+ CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE,
+ OPTION_PATHONLY, OPTION_DIRECTORY, OPTION_HUMAN,
+ OPTION_HIDENONPRINTABLE, OPTION_RECURSIVE, OPTION_REVERSE,
OPTION_MTIME, OPTION_SIZE, OPTION_ATIME);
cf.parse(args);
pathOnly = cf.getOpt(OPTION_PATHONLY);
dirRecurse = !cf.getOpt(OPTION_DIRECTORY);
setRecursive(cf.getOpt(OPTION_RECURSIVE) && dirRecurse);
humanReadable = cf.getOpt(OPTION_HUMAN);
+ hideNonPrintable = cf.getOpt(OPTION_HIDENONPRINTABLE);
orderReverse = cf.getOpt(OPTION_REVERSE);
orderTime = cf.getOpt(OPTION_MTIME);
orderSize = !orderTime && cf.getOpt(OPTION_SIZE);
@@ -163,6 +172,11 @@ class Ls extends FsCommand {
return this.humanReadable;
}
+ @InterfaceAudience.Private
+ private boolean isHideNonPrintable() {
+ return hideNonPrintable;
+ }
+
/**
* Should directory contents be displayed in reverse order
* @return true reverse order, false default order
@@ -241,7 +255,7 @@ class Ls extends FsCommand {
dateFormat.format(new Date(isUseAtime()
? stat.getAccessTime()
: stat.getModificationTime())),
- item);
+ isHideNonPrintable() ? new PrintableString(item.toString()) : item);
out.println(line);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0accc330/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PrintableString.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PrintableString.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PrintableString.java
new file mode 100644
index 0000000..df68f40
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PrintableString.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The {code PrintableString} class converts any string to a printable string
+ * by replacing non-printable characters with ?.
+ *
+ * Categories of Unicode non-printable characters:
+ * <ul>
+ * <li> Control characters (Cc)
+ * <li> Formatting Unicode (Cf)
+ * <li> Private use Unicode (Co)
+ * <li> Unassigned Unicode (Cn)
+ * <li> Standalone surrogate (Unfortunately no matching Unicode category)
+ * </ul>
+ *
+ * @see Character
+ * @see <a href="http://www.unicode.org/">The Unicode Consortium</a>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+class PrintableString {
+ private static final char REPLACEMENT_CHAR = '?';
+
+ private final String printableString;
+
+ PrintableString(String rawString) {
+ StringBuilder stringBuilder = new StringBuilder(rawString.length());
+ for (int offset = 0; offset < rawString.length();) {
+ int codePoint = rawString.codePointAt(offset);
+ offset += Character.charCount(codePoint);
+
+ switch (Character.getType(codePoint)) {
+ case Character.CONTROL: // Cc
+ case Character.FORMAT: // Cf
+ case Character.PRIVATE_USE: // Co
+ case Character.SURROGATE: // Cs
+ case Character.UNASSIGNED: // Cn
+ stringBuilder.append(REPLACEMENT_CHAR);
+ break;
+ default:
+ stringBuilder.append(Character.toChars(codePoint));
+ break;
+ }
+ }
+ printableString = stringBuilder.toString();
+ }
+
+ public String toString() {
+ return printableString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0accc330/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 5790bb7..1723426 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -384,13 +384,14 @@ Return usage output.
ls
----
-Usage: `hadoop fs -ls [-C] [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
+Usage: `hadoop fs -ls [-C] [-d] [-h] [-q] [-R] [-t] [-S] [-r] [-u] <args> `
Options:
* -C: Display the paths of files and directories only.
* -d: Directories are listed as plain files.
* -h: Format file sizes in a human-readable fashion (eg 64.0m instead of 67108864).
+* -q: Print ? instead of non-printable characters.
* -R: Recursively list subdirectories encountered.
* -t: Sort output by modification time (most recent first).
* -S: Sort output by file size.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0accc330/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellList.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellList.java
new file mode 100644
index 0000000..03720d3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellList.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test FsShell -ls command.
+ */
+public class TestFsShellList {
+ private static Configuration conf;
+ private static FsShell shell;
+ private static LocalFileSystem lfs;
+ private static Path testRootDir;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ conf = new Configuration();
+ shell = new FsShell(conf);
+ lfs = FileSystem.getLocal(conf);
+ lfs.setVerifyChecksum(true);
+ lfs.setWriteChecksum(true);
+
+ String root = System.getProperty("test.build.data", "test/build/data");
+ testRootDir = lfs.makeQualified(new Path(root, "testFsShellList"));
+ assertThat(lfs.mkdirs(testRootDir), is(true));
+ }
+
+ @AfterClass
+ public static void teardown() throws Exception {
+ lfs.delete(testRootDir, true);
+ }
+
+ private void createFile(Path filePath) throws Exception {
+ FSDataOutputStream out = lfs.create(filePath);
+ out.writeChars("I am " + filePath);
+ out.close();
+ assertThat(lfs.exists(lfs.getChecksumFile(filePath)), is(true));
+ }
+
+ @Test
+ public void testList() throws Exception {
+ createFile(new Path(testRootDir, "abc"));
+ String[] lsArgv = new String[]{"-ls", testRootDir.toString()};
+ assertThat(shell.run(lsArgv), is(0));
+
+ createFile(new Path(testRootDir, "abc\bd\tef"));
+ createFile(new Path(testRootDir, "ghi"));
+ createFile(new Path(testRootDir, "qq\r123"));
+ lsArgv = new String[]{"-ls", testRootDir.toString()};
+ assertThat(shell.run(lsArgv), is(0));
+
+ lsArgv = new String[]{"-ls", "-q", testRootDir.toString()};
+ assertThat(shell.run(lsArgv), is(0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0accc330/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPrintableString.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPrintableString.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPrintableString.java
new file mode 100644
index 0000000..8e09fc2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPrintableString.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+import org.junit.Test;
+
+/**
+ * Test {@code PrintableString} class.
+ */
+public class TestPrintableString {
+
+ private void expect(String reason, String raw, String expected) {
+ assertThat(reason, new PrintableString(raw).toString(), is(expected));
+ }
+
+ /**
+ * Test printable characters.
+ */
+ @Test
+ public void testPrintableCharacters() throws Exception {
+ // ASCII
+ expect("Should keep ASCII letter", "abcdef237", "abcdef237");
+ expect("Should keep ASCII symbol", " !\"|}~", " !\"|}~");
+
+ // Unicode BMP
+ expect("Should keep Georgian U+1050 and Box Drawing U+2533",
+ "\u1050\u2533--", "\u1050\u2533--");
+
+ // Unicode SMP
+ expect("Should keep Linear B U+10000 and Phoenician U+10900",
+ "\uD800\uDC00'''\uD802\uDD00", "\uD800\uDC00'''\uD802\uDD00");
+ }
+
+ /**
+ * Test non-printable characters.
+ */
+ @Test
+ public void testNonPrintableCharacters() throws Exception {
+ // Control characters
+ expect("Should replace single control character", "abc\rdef", "abc?def");
+ expect("Should replace multiple control characters",
+ "\babc\tdef", "?abc?def");
+ expect("Should replace all control characters", "\f\f\b\n", "????");
+ expect("Should replace mixed characters starting with a control",
+ "\027ab\0", "?ab?");
+
+ // Formatting Unicode
+ expect("Should replace Byte Order Mark", "-\uFEFF--", "-?--");
+ expect("Should replace Invisible Separator", "\u2063\t", "??");
+
+ // Private use Unicode
+ expect("Should replace private use U+E000", "\uE000", "?");
+ expect("Should replace private use U+E123 and U+F432",
+ "\uE123abc\uF432", "?abc?");
+ expect("Should replace private use in Plane 15 and 16: U+F0000 and " +
+ "U+10FFFD, but keep U+1050",
+ "x\uDB80\uDC00y\uDBFF\uDFFDz\u1050", "x?y?z\u1050");
+
+ // Unassigned Unicode
+ expect("Should replace unassigned U+30000 and U+DFFFF",
+ "-\uD880\uDC00-\uDB3F\uDFFF-", "-?-?-");
+
+ // Standalone surrogate character (not in a pair)
+ expect("Should replace standalone surrogate U+DB80", "x\uDB80yz", "x?yz");
+ expect("Should replace standalone surrogate mixed with valid pair",
+ "x\uDB80\uD802\uDD00yz", "x?\uD802\uDD00yz");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0accc330/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/package-info.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/package-info.java
new file mode 100644
index 0000000..47a4e7a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/package-info.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package for {@code org.apache.hadoop.fs.shell} test classes.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.shell;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0accc330/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 3ad6d66..bbbc1ec 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -54,7 +54,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-ls \[-C\] \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)*</expected-output>
+ <expected-output>^-ls \[-C\] \[-d\] \[-h\] \[-q\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -106,6 +106,10 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
+ <expected-output>^\s*-q\s+Print \? instead of non-printable characters\.( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
<expected-output>^\s*rather than a number of bytes\.( )*</expected-output>
</comparator>
<comparator>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org