You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/05/30 02:16:27 UTC
hadoop git commit: HADOOP-12043. Display warning if defaultFs is not
set when running fs commands. Contributed by Eddy Xu.
Repository: hadoop
Updated Branches:
refs/heads/trunk 6aec13cb3 -> 374ddd9f9
HADOOP-12043. Display warning if defaultFs is not set when running fs commands. Contributed by Eddy Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/374ddd9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/374ddd9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/374ddd9f
Branch: refs/heads/trunk
Commit: 374ddd9f9ea43b0e730a7baab3e51e6893d40420
Parents: 6aec13c
Author: Andrew Wang <wa...@apache.org>
Authored: Fri May 29 17:15:58 2015 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri May 29 17:15:58 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../fs/CommonConfigurationKeysPublic.java | 6 ++++
.../org/apache/hadoop/fs/shell/FsCommand.java | 30 +++++++++++++++++++-
.../java/org/apache/hadoop/fs/shell/Ls.java | 8 ++++++
.../src/main/resources/core-default.xml | 8 ++++++
.../java/org/apache/hadoop/fs/shell/TestLs.java | 25 ++++++++++++++++
6 files changed, 79 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e5d2e39..00075b4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -626,6 +626,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating
(Masatake Iwasaki via Colin P. McCabe)
+ HADOOP-12043. Display warning if defaultFs is not set when running fs
+ commands. (Lei Xu via wang)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 90c6934..7231d59 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -363,5 +363,11 @@ public class CommonConfigurationKeysPublic {
"hadoop.security.random.device.file.path";
public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT =
"/dev/urandom";
+
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY =
+ "hadoop.shell.missing.defaultFs.warning";
+ public static final boolean HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT =
+ false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
index 9515fde..971950f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.shell;
import java.io.IOException;
+import java.util.LinkedList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -27,8 +28,13 @@ import org.apache.hadoop.fs.FsShellPermissions;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.find.Find;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
+
/**
- * Base class for all "hadoop fs" commands
+ * Base class for all "hadoop fs" commands.
*/
@InterfaceAudience.Private
@@ -90,4 +96,26 @@ abstract public class FsCommand extends Command {
public int runAll() {
return run(args);
}
+
+ @Override
+ protected void processRawArguments(LinkedList<String> args)
+ throws IOException {
+ LinkedList<PathData> expendedArgs = expandArguments(args);
+ // If "fs.defaultFs" is not set appropriately, it warns the user that the
+ // command is not running against HDFS.
+ final boolean displayWarnings = getConf().getBoolean(
+ HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY,
+ HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT);
+ if (displayWarnings) {
+ final String defaultFs = getConf().get(FS_DEFAULT_NAME_KEY);
+ final boolean missingDefaultFs =
+ defaultFs == null || defaultFs.equals(FS_DEFAULT_NAME_DEFAULT);
+ if (missingDefaultFs) {
+ err.printf(
+ "Warning: fs.defaultFs is not set when running \"%s\" command.%n",
+ getCommandName());
+ }
+ }
+ processArguments(expendedArgs);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index d5c52ed..67348c6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -24,6 +24,8 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.LinkedList;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -102,6 +104,12 @@ class Ls extends FsCommand {
protected boolean humanReadable = false;
+ protected Ls() {}
+
+ protected Ls(Configuration conf) {
+ super(conf);
+ }
+
protected String formatSize(long size) {
return humanReadable
? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index a1bc780..ae23a75 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1899,4 +1899,12 @@ for ldap providers in the same way as above does.
<value>Client</value>
</property>
+ <property>
+ <description>
+ Enable hdfs shell commands to display warnings if (fs.defaultFS) property
+ is not set.
+ </description>
+ <name>hadoop.shell.missing.defaultFs.warning</name>
+ <value>false</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/374ddd9f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
index 4a9103f..cba44ee 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
@@ -17,10 +17,12 @@
*/
package org.apache.hadoop.fs.shell;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
import static org.junit.Assert.*;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.*;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
@@ -1048,6 +1050,29 @@ public class TestLs {
verifyNoMoreInteractions(out);
}
+ private static void displayWarningOnLocalFileSystem(boolean shouldDisplay)
+ throws IOException {
+ Configuration conf = new Configuration();
+ conf.setBoolean(
+ HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, shouldDisplay);
+
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+ PrintStream err = new PrintStream(buf, true);
+ Ls ls = new Ls(conf);
+ ls.err = err;
+ ls.run("file:///.");
+ assertEquals(shouldDisplay, buf.toString().contains(
+ "Warning: fs.defaultFs is not set when running \"ls\" command."));
+ }
+
+ @Test
+ public void displayWarningsOnLocalFileSystem() throws IOException {
+ // Display warnings.
+ displayWarningOnLocalFileSystem(true);
+ // Does not display warnings.
+ displayWarningOnLocalFileSystem(false);
+ }
+
// check the deprecated flag isn't set
@Test
public void isDeprecated() {