You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2011/03/29 02:39:59 UTC
svn commit: r1086455 - in /hadoop/common/trunk: ./
src/java/org/apache/hadoop/fs/ src/java/org/apache/hadoop/fs/shell/
src/test/core/org/apache/hadoop/fs/shell/
Author: szetszwo
Date: Tue Mar 29 00:39:58 2011
New Revision: 1086455
URL: http://svn.apache.org/viewvc?rev=1086455&view=rev
Log:
Revert HADOOP-7202 since HDFS cannot be compiled.
Removed:
hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/FsCommand.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/PathData.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/shell/
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Command.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Count.java
Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=1086455&r1=1086454&r2=1086455&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Tue Mar 29 00:39:58 2011
@@ -84,8 +84,6 @@ Trunk (unreleased changes)
HADOOP-7180. Better support on CommandFormat on the API and exceptions.
(Daryn Sharp via szetszwo)
- HADOOP-7202. Improve shell Command base class. (Daryn Sharp via szetszwo)
-
OPTIMIZATIONS
HADOOP-6949. Reduce RPC packet size of primitive arrays using
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java?rev=1086455&r1=1086454&r2=1086455&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java Tue Mar 29 00:39:58 2011
@@ -370,7 +370,7 @@ public class FsShell extends Configured
private class TextRecordInputStream extends InputStream {
SequenceFile.Reader r;
- WritableComparable<?> key;
+ WritableComparable key;
Writable val;
DataInputBuffer inbuf;
@@ -1691,6 +1691,8 @@ public class FsShell extends Configured
delete(argv[i], true, rmSkipTrash);
} else if ("-df".equals(cmd)) {
df(argv[i]);
+ } else if (Count.matches(cmd)) {
+ new Count(argv, i, getConf()).runAll();
} else if ("-ls".equals(cmd)) {
exitCode = ls(argv[i], false);
} else if ("-lsr".equals(cmd)) {
@@ -1757,7 +1759,7 @@ public class FsShell extends Configured
} else if ("-df".equals(cmd) ) {
System.err.println("Usage: java FsShell" +
" [" + cmd + " [<path>]]");
- } else if ("-count".equals(cmd)) {
+ } else if (Count.matches(cmd)) {
System.err.println(prefix + " [" + Count.USAGE + "]");
} else if ("-rm".equals(cmd) || "-rmr".equals(cmd)) {
System.err.println("Usage: java FsShell [" + cmd +
@@ -1946,13 +1948,8 @@ public class FsShell extends Configured
du(argv, i);
} else if ("-dus".equals(cmd)) {
dus(argv, i);
- } else if ("-count".equals(cmd)) {
- // TODO: next two lines are a temporary crutch until this entire
- // block is overhauled
- LinkedList<String> args = new LinkedList<String>(Arrays.asList(argv));
- args.removeFirst();
- Count runner = ReflectionUtils.newInstance(Count.class, getConf());
- exitCode = runner.run(cmd, args);
+ } else if (Count.matches(cmd)) {
+ exitCode = new Count(argv, i, getConf()).runAll();
} else if ("-mkdir".equals(cmd)) {
exitCode = doall(cmd, argv, i);
} else if ("-touchz".equals(cmd)) {
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Command.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Command.java?rev=1086455&r1=1086454&r2=1086455&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Command.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Command.java Tue Mar 29 00:39:58 2011
@@ -17,53 +17,27 @@
*/
package org.apache.hadoop.fs.shell;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
+import java.io.*;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.ipc.RemoteException;
/**
* An abstract class for the execution of a file system command
*/
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-
abstract public class Command extends Configured {
protected String[] args;
- protected String cmdName;
- protected int exitCode = 0;
- protected int numErrors = 0;
- protected boolean recursive = false;
- protected ArrayList<Exception> exceptions = new ArrayList<Exception>();
-
- private static final Log LOG = LogFactory.getLog(Command.class);
-
- /** allows stdout to be captured if necessary */
- public PrintStream out = System.out;
- /** allows stderr to be captured if necessary */
- public PrintStream err = System.err;
-
- protected Command() {}
/** Constructor */
protected Command(Configuration conf) {
super(conf);
}
- /** @return the command's name excluding the leading character - */
+ /** Return the command's name excluding the leading character - */
abstract public String getCommandName();
/**
@@ -83,273 +57,30 @@ abstract public class Command extends Co
int exitCode = 0;
for (String src : args) {
try {
- List<PathData> srcs = expandGlob(src);
- for(PathData s : srcs) {
- run(s.path);
+ Path srcPath = new Path(src);
+ FileSystem fs = srcPath.getFileSystem(getConf());
+ FileStatus[] statuses = fs.globStatus(srcPath);
+ if (statuses == null) {
+ System.err.println("Can not find listing for " + src);
+ exitCode = -1;
+ } else {
+ for(FileStatus s : statuses) {
+ run(s.getPath());
+ }
}
- } catch (IOException e) {
+ } catch (RemoteException re) {
exitCode = -1;
- displayError(e);
- }
- }
- return exitCode;
- }
-
- /**
- * Invokes the command handler. The default behavior is to process options,
- * expand arguments, and then process each argument.
- * <pre>
- * run
- * \-> {@link #processOptions(LinkedList)}
- * \-> {@link #expandArguments(LinkedList)} -> {@link #expandArgument(String)}*
- * \-> {@link #processArguments(LinkedList)}
- * \-> {@link #processArgument(PathData)}*
- * \-> {@link #processPathArgument(PathData)}
- * \-> {@link #processPaths(PathData, PathData...)}
- * \-> {@link #processPath(PathData)}*
- * \-> {@link #processNonexistentPathArgument(PathData)}
- * </pre>
- * Most commands will chose to implement just
- * {@link #processOptions(LinkedList)} and {@link #processPath(PathData)}
- *
- * @param cmd the command's name
- * @param args the list of command line arguments
- * @return the exit code for the command
- * @throws IllegalArgumentException if called with invalid arguments
- */
- public int run(String cmd, LinkedList<String> args) {
- cmdName = cmd;
- try {
- processOptions(args);
- processArguments(expandArguments(args));
- } catch (IOException e) {
- displayError(e);
- }
-
- return (numErrors == 0) ? exitCode : 1;
- }
-
- /**
- * Must be implemented by commands to process the command line flags and
- * check the bounds of the remaining arguments. If an
- * IllegalArgumentException is thrown, the FsShell object will print the
- * short usage of the command.
- * @param args the command line arguments
- * @throws IOException
- */
- protected void processOptions(LinkedList<String> args) throws IOException {}
-
- /**
- * Expands a list of arguments into {@link PathData} objects. The default
- * behavior is to call {@link #expandArgument(String)} on each element
- * which by default globs the argument. The loop catches IOExceptions,
- * increments the error count, and displays the exception.
- * @param args strings to expand into {@link PathData} objects
- * @return list of all {@link PathData} objects the arguments
- * @throws IOException if anything goes wrong...
- */
- protected LinkedList<PathData> expandArguments(LinkedList<String> args)
- throws IOException {
- LinkedList<PathData> expandedArgs = new LinkedList<PathData>();
- for (String arg : args) {
- try {
- expandedArgs.addAll(expandArgument(arg));
- } catch (IOException e) { // other exceptions are probably nasty
- displayError(e);
- }
- }
- return expandedArgs;
- }
-
- /**
- * Expand the given argument into a list of {@link PathData} objects.
- * The default behavior is to expand globs. Commands may override to
- * perform other expansions on an argument.
- * @param arg string pattern to expand
- * @return list of {@link PathData} objects
- * @throws IOException if anything goes wrong...
- */
- protected List<PathData> expandArgument(String arg) throws IOException {
- return expandGlob(arg);
- }
-
- /**
- * Processes the command's list of expanded arguments.
- * {@link #processArgument(PathData)} will be invoked with each item
- * in the list. The loop catches IOExceptions, increments the error
- * count, and displays the exception.
- * @param args a list of {@link PathData} to process
- * @throws IOException if anything goes wrong...
- */
- protected void processArguments(LinkedList<PathData> args)
- throws IOException {
- for (PathData arg : args) {
- try {
- processArgument(arg);
- } catch (IOException e) {
- displayError(e);
- }
- }
- }
-
- /**
- * Processes a {@link PathData} item, calling
- * {@link #processPathArgument(PathData)} or
- * {@link #processNonexistentPathArgument(PathData)} on each item.
- * @param arg {@link PathData} item to process
- * @throws IOException if anything goes wrong...
- */
- protected void processArgument(PathData item) throws IOException {
- if (item.exists) {
- processPathArgument(item);
- } else {
- processNonexistentPathArgument(item);
- }
- }
-
- /**
- * This is the last chance to modify an argument before going into the
- * (possibly) recursive {@link #processPaths(PathData, PathData...)}
- * -> {@link #processPath(PathData)} loop. Ex. ls and du use this to
- * expand out directories.
- * @param item a {@link PathData} representing a path which exists
- * @throws IOException if anything goes wrong...
- */
- protected void processPathArgument(PathData item) throws IOException {
- // null indicates that the call is not via recursion, ie. there is
- // no parent directory that was expanded
- processPaths(null, item);
- }
-
- /**
- * Provides a hook for handling paths that don't exist. By default it
- * will throw an exception. Primarily overriden by commands that create
- * paths such as mkdir or touch.
- * @param item the {@link PathData} that doesn't exist
- * @throws FileNotFoundException if arg is a path and it doesn't exist
- * @throws IOException if anything else goes wrong...
- */
- protected void processNonexistentPathArgument(PathData item)
- throws IOException {
- throw new FileNotFoundException(item + ": No such file or directory");
- }
-
- /**
- * Iterates over the given expanded paths and invokes
- * {@link #processPath(PathData)} on each element. If "recursive" is true,
- * will do a post-visit DFS on directories.
- * @param parent if called via a recurse, will be the parent dir, else null
- * @param items a list of {@link PathData} objects to process
- * @throws IOException if anything goes wrong...
- */
- protected void processPaths(PathData parent, PathData ... items)
- throws IOException {
- // TODO: this really should be iterative
- for (PathData item : items) {
- try {
- processPath(item);
- if (recursive && item.stat.isDirectory()) {
- recursePath(item);
+ String content = re.getLocalizedMessage();
+ int eol = content.indexOf('\n');
+ if (eol>=0) {
+ content = content.substring(0, eol);
}
+ System.err.println(getCommandName() + ": " + content);
} catch (IOException e) {
- displayError(e);
- }
- }
- }
-
- /**
- * Hook for commands to implement an operation to be applied on each
- * path for the command. Note implementation of this method is optional
- * if earlier methods in the chain handle the operation.
- * @param item a {@link PathData} object
- * @throws RuntimeException if invoked but not implemented
- * @throws IOException if anything else goes wrong in the user-implementation
- */
- protected void processPath(PathData item) throws IOException {
- throw new RuntimeException("processPath() is not implemented");
- }
-
- /**
- * Gets the directory listing for a path and invokes
- * {@link #processPaths(PathData, PathData...)}
- * @param item {@link PathData} for directory to recurse into
- * @throws IOException if anything goes wrong...
- */
- protected void recursePath(PathData item) throws IOException {
- processPaths(item, item.getDirectoryContents());
- }
-
- /**
- * Expand the given path as a glob pattern. Non-existent paths do not
- * throw an exception because creation commands like touch and mkdir need
- * to create them. The "stat" field will be null if the path does not
- * exist.
- * @param pattern the glob to expand
- * @return list of {@link PathData} objects
- * @throws FileNotFoundException the path is a glob with no matches
- * @throws IOException anything else goes wrong...
- */
- protected List<PathData> expandGlob(String pattern) throws IOException {
- Path path = new Path(pattern);
- FileSystem fs = path.getFileSystem(getConf());
- FileStatus[] stats = fs.globStatus(path);
-
- if (stats != null && stats.length == 0) { // glob failed to match
- throw new FileNotFoundException(path + ": No such file or directory");
- }
-
- List<PathData> items = new LinkedList<PathData>();
- if (stats == null) { // not a glob & file not found, so null stat block
- items.add(new PathData(fs, path, null));
- } else {
- // convert all the stats to PathData objs
- for (FileStatus stat : stats) {
- items.add(new PathData(fs, stat));
+ exitCode = -1;
+ System.err.println(getCommandName() + ": " + e.getLocalizedMessage());
}
}
- return items;
- }
-
- /**
- * Display an exception prefaced with the command name. Also increments
- * the error count for the command which will result in a non-zero exit
- * code.
- * @param e exception to display
- */
- public void displayError(Exception e) {
- // build up a list of exceptions that occurred
- exceptions.add(e);
-
- String errorMessage = e.getLocalizedMessage();
- if (errorMessage == null) {
- // this is an unexpected condition, so dump the whole exception since
- // it's probably a nasty internal error where the backtrace would be
- // useful
- errorMessage = StringUtils.stringifyException(e);
- LOG.debug(errorMessage);
- } else {
- errorMessage = errorMessage.split("\n", 2)[0];
- }
- displayError(errorMessage);
- }
-
- /**
- * Display an error string prefaced with the command name. Also increments
- * the error count for the command which will result in a non-zero exit
- * code.
- * @param message error message to display
- */
- public void displayError(String message) {
- numErrors++;
- displayWarning(message);
- }
-
- /**
- * Display an warning string prefaced with the command name.
- * @param message warning message to display
- */
- public void displayWarning(String message) {
- String cmd = cmdName.startsWith("-") ? cmdName.substring(1) : cmdName;
- err.println(cmd + ": " + message);
+ return exitCode;
}
}
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Count.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Count.java?rev=1086455&r1=1086454&r2=1086455&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Count.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Count.java Tue Mar 29 00:39:58 2011
@@ -17,22 +17,19 @@
*/
package org.apache.hadoop.fs.shell;
-import java.io.IOException;
-import java.util.LinkedList;
+import java.io.*;
+import java.util.List;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
/**
* Count the number of directories, files, bytes, quota, and remaining quota.
*/
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-
-public class Count extends FsCommand {
+public class Count extends Command {
public static final String NAME = "count";
- public static final String USAGE = "-" + NAME + " [-q] <path> ...";
+ public static final String USAGE = "-" + NAME + "[-q] <path>";
public static final String DESCRIPTION = CommandUtils.formatDescription(USAGE,
"Count the number of directories, files and bytes under the paths",
"that match the specified file pattern. The output columns are:",
@@ -40,21 +37,41 @@ public class Count extends FsCommand {
"QUOTA REMAINING_QUATA SPACE_QUOTA REMAINING_SPACE_QUOTA ",
" DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME");
- private boolean showQuotas;
+ private boolean qOption;
- @Override
- protected void processOptions(LinkedList<String> args) {
- CommandFormat cf = new CommandFormat(NAME, 1, Integer.MAX_VALUE, "q");
- cf.parse(args);
- if (args.size() == 0) { // default path is the current working directory
- args.add(".");
+ /** Constructor
+ *
+ * @param cmd the count command
+ * @param pos the starting index of the arguments
+ */
+ public Count(String[] cmd, int pos, Configuration conf) {
+ super(conf);
+ CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE, "q");
+ List<String> parameters = c.parse(cmd, pos);
+ this.args = parameters.toArray(new String[parameters.size()]);
+ if (this.args.length == 0) { // default path is the current working directory
+ this.args = new String[] {"."};
}
- showQuotas = cf.getOpt("q");
+ this.qOption = c.getOpt("q") ? true: false;
+ }
+
+ /** Check if a command is the count command
+ *
+ * @param cmd A string representation of a command starting with "-"
+ * @return true if this is a count command; false otherwise
+ */
+ public static boolean matches(String cmd) {
+ return ("-" + NAME).equals(cmd);
+ }
+
+ @Override
+ public String getCommandName() {
+ return NAME;
}
@Override
- protected void processPath(PathData src) throws IOException {
- ContentSummary summary = src.fs.getContentSummary(src.path);
- out.println(summary.toString(showQuotas) + src.path);
+ protected void run(Path path) throws IOException {
+ FileSystem fs = path.getFileSystem(getConf());
+ System.out.println(fs.getContentSummary(path).toString(qOption) + path);
}
}