You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2011/05/23 23:46:50 UTC
svn commit: r1126769 - in /hadoop/common/branches/yahoo-merge: ./
src/contrib/ec2/ src/docs/ src/java/ src/java/org/apache/hadoop/fs/
src/java/org/apache/hadoop/fs/shell/ src/test/core/
src/test/core/org/apache/hadoop/fs/shell/
Author: szetszwo
Date: Mon May 23 21:46:49 2011
New Revision: 1126769
URL: http://svn.apache.org/viewvc?rev=1126769&view=rev
Log:
Merge 1091902 and 1091970 from trunk for HADOOP-7224.
Added:
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/CommandFactory.java
- copied unchanged from r1091902, hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/CommandFactory.java
hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/shell/TestCommandFactory.java
- copied unchanged from r1091970, hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/shell/TestCommandFactory.java
Modified:
hadoop/common/branches/yahoo-merge/ (props changed)
hadoop/common/branches/yahoo-merge/CHANGES.txt (contents, props changed)
hadoop/common/branches/yahoo-merge/src/contrib/ec2/ (props changed)
hadoop/common/branches/yahoo-merge/src/docs/ (props changed)
hadoop/common/branches/yahoo-merge/src/java/ (props changed)
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Command.java
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Count.java
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
hadoop/common/branches/yahoo-merge/src/test/core/ (props changed)
Propchange: hadoop/common/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1100026
+/hadoop/common/trunk:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1100026
/hadoop/core/branches/branch-0.19/core:713112
Modified: hadoop/common/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/CHANGES.txt?rev=1126769&r1=1126768&r2=1126769&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/common/branches/yahoo-merge/CHANGES.txt Mon May 23 21:46:49 2011
@@ -27,6 +27,8 @@ Trunk (unreleased changes)
HADOOP-7202. Improve shell Command base class. (Daryn Sharp via szetszwo)
+ HADOOP-7224. Add CommandFactory to shell. (Daryn Sharp via szetszwo)
+
OPTIMIZATIONS
BUG FIXES
Propchange: hadoop/common/branches/yahoo-merge/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/CHANGES.txt:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1092832,1100026
+/hadoop/common/trunk/CHANGES.txt:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1092832,1100026
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
/hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278
Propchange: hadoop/common/branches/yahoo-merge/src/contrib/ec2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/contrib/ec2:1043117,1078148,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1100026
+/hadoop/common/trunk/src/contrib/ec2:1043117,1078148,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1100026
/hadoop/core/branches/branch-0.19/core/src/contrib/ec2:713112
/hadoop/core/trunk/src/contrib/ec2:776175-784663
Propchange: hadoop/common/branches/yahoo-merge/src/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/src/docs:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1100026
+/hadoop/common/trunk/src/docs:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1100026
/hadoop/core/branches/branch-0.19/src/docs:713112
Propchange: hadoop/common/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/java:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1100026
+/hadoop/common/trunk/src/java:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1100026
/hadoop/core/branches/branch-0.19/core/src/java:713112
/hadoop/core/trunk/src/core:776175-785643,785929-786278
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java?rev=1126769&r1=1126768&r2=1126769&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java Mon May 23 21:46:49 2011
@@ -26,7 +26,6 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
-import java.util.LinkedList;
import java.util.List;
import java.util.TimeZone;
import java.util.zip.GZIPInputStream;
@@ -34,8 +33,10 @@ import java.util.zip.GZIPInputStream;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.shell.Command;
+import org.apache.hadoop.fs.shell.CommandFactory;
import org.apache.hadoop.fs.shell.CommandFormat;
-import org.apache.hadoop.fs.shell.Count;
+import org.apache.hadoop.fs.shell.FsCommand;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.IOUtils;
@@ -57,6 +58,8 @@ public class FsShell extends Configured
protected FileSystem fs;
private Trash trash;
+ protected CommandFactory commandFactory;
+
public static final SimpleDateFormat dateForm =
new SimpleDateFormat("yyyy-MM-dd HH:mm");
protected static final SimpleDateFormat modifFmt =
@@ -82,6 +85,7 @@ public class FsShell extends Configured
super(conf);
fs = null;
trash = null;
+ commandFactory = new CommandFactory();
}
protected void init() throws IOException {
@@ -1385,9 +1389,7 @@ public class FsShell extends Configured
"[-tail [-f] <path>] [-text <path>]\n\t" +
"[" + FsShellPermissions.CHMOD_USAGE + "]\n\t" +
"[" + FsShellPermissions.CHOWN_USAGE + "]\n\t" +
- "[" + FsShellPermissions.CHGRP_USAGE + "]\n\t" +
- "[" + Count.USAGE + "]\n\t" +
- "[-help [cmd]]\n";
+ "[" + FsShellPermissions.CHGRP_USAGE + "]";
String conf ="-conf <configuration file>: Specify an application configuration file.";
@@ -1546,7 +1548,10 @@ public class FsShell extends Configured
String help = "-help [cmd]: \tDisplays help for given command or all commands if none\n" +
"\t\tis specified.\n";
- if ("fs".equals(cmd)) {
+ Command instance = commandFactory.getInstance("-" + cmd);
+ if (instance != null) {
+ System.out.println(instance.getDescription());
+ } else if ("fs".equals(cmd)) {
System.out.println(fs);
} else if ("conf".equals(cmd)) {
System.out.println(conf);
@@ -1610,12 +1615,16 @@ public class FsShell extends Configured
System.out.println(chown);
} else if ("chgrp".equals(cmd)) {
System.out.println(chgrp);
- } else if (Count.NAME.equals(cmd)) {
- System.out.println(Count.DESCRIPTION);
} else if ("help".equals(cmd)) {
System.out.println(help);
} else {
System.out.println(summary);
+ for (String thisCmdName : commandFactory.getNames()) {
+ instance = commandFactory.getInstance(thisCmdName);
+ System.out.println(instance.getUsage());
+ }
+ System.out.println("\t[-help [cmd]]\n");
+
System.out.println(fs);
System.out.println(ls);
System.out.println(lsr);
@@ -1644,7 +1653,12 @@ public class FsShell extends Configured
System.out.println(chmod);
System.out.println(chown);
System.out.println(chgrp);
- System.out.println(Count.DESCRIPTION);
+
+ for (String thisCmdName : commandFactory.getNames()) {
+ instance = commandFactory.getInstance(thisCmdName);
+ System.out.println(instance.getDescription());
+ }
+
System.out.println(help);
}
}
@@ -1727,9 +1741,13 @@ public class FsShell extends Configured
* Displays format of commands.
*
*/
- private static void printUsage(String cmd) {
+ private void printUsage(String cmd) {
String prefix = "Usage: java " + FsShell.class.getSimpleName();
- if ("-fs".equals(cmd)) {
+
+ Command instance = commandFactory.getInstance(cmd);
+ if (instance != null) {
+ System.err.println(prefix + " [" + instance.getUsage() + "]");
+ } else if ("-fs".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [-fs <local | file system URI>]");
} else if ("-conf".equals(cmd)) {
@@ -1747,8 +1765,6 @@ public class FsShell extends Configured
} else if ("-df".equals(cmd) ) {
System.err.println("Usage: java FsShell" +
" [" + cmd + " [<path>]]");
- } else if ("-count".equals(cmd)) {
- System.err.println(prefix + " [" + Count.USAGE + "]");
} else if ("-rm".equals(cmd) || "-rmr".equals(cmd)) {
System.err.println("Usage: java FsShell [" + cmd +
" [-skipTrash] <src>]");
@@ -1786,7 +1802,6 @@ public class FsShell extends Configured
System.err.println(" [-df [<path>]]");
System.err.println(" [-du [-s] [-h] <path>]");
System.err.println(" [-dus <path>]");
- System.err.println(" [" + Count.USAGE + "]");
System.err.println(" [-mv <src> <dst>]");
System.err.println(" [-cp <src> <dst>]");
System.err.println(" [-rm [-skipTrash] <path>]");
@@ -1810,6 +1825,10 @@ public class FsShell extends Configured
System.err.println(" [" + FsShellPermissions.CHMOD_USAGE + "]");
System.err.println(" [" + FsShellPermissions.CHOWN_USAGE + "]");
System.err.println(" [" + FsShellPermissions.CHGRP_USAGE + "]");
+ for (String name : commandFactory.getNames()) {
+ instance = commandFactory.getInstance(name);
+ System.err.println(" [" + instance.getUsage() + "]");
+ }
System.err.println(" [-help [cmd]]");
System.err.println();
ToolRunner.printGenericCommandUsage(System.err);
@@ -1820,7 +1839,12 @@ public class FsShell extends Configured
* run
*/
public int run(String argv[]) throws Exception {
-
+ // TODO: This isn't the best place, but this class is being abused with
+ // subclasses which of course override this method. There really needs
+ // to be a better base class for all commands
+ commandFactory.setConf(getConf());
+ commandFactory.registerCommands(FsCommand.class);
+
if (argv.length < 1) {
printUsage("");
return -1;
@@ -1873,7 +1897,19 @@ public class FsShell extends Configured
exitCode = 0;
try {
- if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) {
+ Command instance = commandFactory.getInstance(cmd);
+ if (instance != null) {
+ try {
+ exitCode = instance.run(Arrays.copyOfRange(argv, i, argv.length));
+ } catch (Exception e) {
+ exitCode = -1;
+ LOG.debug("Error", e);
+ instance.displayError(e);
+ if (e instanceof IllegalArgumentException) {
+ printUsage(cmd);
+ }
+ }
+ } else if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) {
Path[] srcs = new Path[argv.length-2];
for (int j=0 ; i < argv.length-1 ;)
srcs[j++] = new Path(argv[i++]);
@@ -1934,12 +1970,6 @@ public class FsShell extends Configured
du(argv, i);
} else if ("-dus".equals(cmd)) {
dus(argv, i);
- } else if ("-count".equals(cmd)) {
- // TODO: next two lines are a temporary crutch until this entire
- // block is overhauled
- Count runner = ReflectionUtils.newInstance(Count.class, getConf());
- runner.setCommandName(cmd); // TODO: will change with factory
- exitCode = runner.run(Arrays.copyOfRange(argv, 1, argv.length));
} else if ("-mkdir".equals(cmd)) {
exitCode = doall(cmd, argv, i);
} else if ("-touchz".equals(cmd)) {
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Command.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Command.java?rev=1126769&r1=1126768&r2=1126769&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Command.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Command.java Mon May 23 21:46:49 2011
@@ -365,4 +365,35 @@ abstract public class Command extends Co
public void displayWarning(String message) {
err.println(getCommandName() + ": " + message);
}
+
+ /**
+ * The short usage suitable for the synopsis
+ * @return "name options"
+ */
+ public String getUsage() {
+ return getCommandField("USAGE");
+ }
+
+ /**
+ * The long usage suitable for help output
+ * @return text of the usage
+ */
+ public String getDescription() {
+ return getCommandField("DESCRIPTION");
+ }
+
+ /**
+ * Get a public static class field
+ * @param field the field to retrieve
+ * @return String of the field
+ */
+ private String getCommandField(String field) {
+ String value;
+ try {
+ value = (String)this.getClass().getField(field).get(null);
+ } catch (Exception e) {
+ throw new RuntimeException(StringUtils.stringifyException(e));
+ }
+ return value;
+ }
}
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Count.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Count.java?rev=1126769&r1=1126768&r2=1126769&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Count.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Count.java Mon May 23 21:46:49 2011
@@ -34,6 +34,14 @@ import org.apache.hadoop.fs.FsShell;
@InterfaceStability.Evolving
public class Count extends FsCommand {
+ /**
+ * Register the names for the count command
+ * @param factory the command factory that will instantiate this class
+ */
+ public static void registerCommands(CommandFactory factory) {
+ factory.addClass(Count.class, "-count");
+ }
+
public static final String NAME = "count";
public static final String USAGE = "-" + NAME + "[-q] <path>";
public static final String DESCRIPTION = CommandUtils.formatDescription(USAGE,
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java?rev=1126769&r1=1126768&r2=1126769&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java Mon May 23 21:46:49 2011
@@ -37,6 +37,14 @@ import org.apache.hadoop.fs.Path;
// used to implement unnecessary abstract methods in the base class
abstract public class FsCommand extends Command {
+ /**
+ * Register the command classes used by the fs subcommand
+ * @param factory where to register the class
+ */
+ public static void registerCommands(CommandFactory factory) {
+ Count.registerCommands(factory);
+ }
+
protected FsCommand() {}
protected FsCommand(Configuration conf) {
Propchange: hadoop/common/branches/yahoo-merge/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 21:46:49 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/test/core:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1092565,1100026
+/hadoop/common/trunk/src/test/core:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092565,1100026
/hadoop/core/branches/branch-0.19/core/src/test/core:713112
/hadoop/core/trunk/src/test/core:776175-785643,785929-786278