You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2007/10/24 23:24:53 UTC
svn commit: r588035 [3/3] - in /lucene/hadoop/branches/branch-0.15: ./
src/contrib/streaming/src/java/org/apache/hadoop/streaming/
src/java/org/apache/hadoop/conf/ src/java/org/apache/hadoop/filecache/
src/java/org/apache/hadoop/io/ src/java/org/apache...
Modified: lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/GenericOptionsParser.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/GenericOptionsParser.java?rev=588035&r1=588034&r2=588035&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/GenericOptionsParser.java (original)
+++ lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/GenericOptionsParser.java Wed Oct 24 14:24:51 2007
@@ -32,50 +32,54 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-/*************************************************************
- * This class can be used as a utility to parse command line
- * arguments generic to the Hadoop framework. This class
- * recognizes several command line arguments, which allow a user
- * to specify a namenode, a job tracker etc. Generic options
- * supported are
- * <p>-conf <configuration file> specify an application configuration file
- * <p>-D <property=value> use value for given property
- * <p>-fs <local|namenode:port> specify a namenode
- * <p>-jt <local|jobtracker:port> specify a job tracker
- * <br>
- * <p>The general command line syntax is
- * <p>bin/hadoop command [genericOptions] [commandOptions]
- * <br>
- * Generic command line arguments <strong>might</strong> modify
- * <code>Configuration </code> objects, given to constructors
- * <br><br>
- * The functionality is implemented using Commons CLI.
- * <br>
- * <p>Examples using generic options are
- * <p>bin/hadoop dfs -fs darwin:8020 -ls /data
- * <p><blockquote><pre>
- * list /data directory in dfs with namenode darwin:8020
- * </pre></blockquote>
- * <p>bin/hadoop dfs -D fs.default.name=darwin:8020 -ls /data
- * <p><blockquote><pre>
- * list /data directory in dfs with namenode darwin:8020
- * </pre></blockquote>
- * <p>bin/hadoop dfs -conf hadoop-site.xml -ls /data
- * <p><blockquote><pre>
- * list /data directory in dfs with conf specified in hadoop-site.xml
- * </pre></blockquote>
- * <p>bin/hadoop job -D mapred.job.tracker=darwin:50020 -submit job.xml
- * <p><blockquote><pre>
- * submit a job to job tracker darwin:50020
- * </pre></blockquote>
- * <p>bin/hadoop job -jt darwin:50020 -submit job.xml
- * <p><blockquote><pre>
- * submit a job to job tracker darwin:50020
- * </pre></blockquote>
- * <p>bin/hadoop job -jt local -submit job.xml
+/**
+ * <code>GenericOptionsParser</code> is a utility to parse command line
+ * arguments generic to the Hadoop framework.
+ *
+ * <code>GenericOptionsParser</code> recognizes several standarad command
+ * line arguments, enabling applications to easily specify a namenode, a
+ * jobtracker, additional configuration resources etc.
+ *
+ * <h4 id="GenericOptions">Generic Options</h4>
+ *
+ * <p>The supported generic options are:</p>
+ * <p><blockquote><pre>
+ * -conf <configuration file> specify a configuration file
+ * -D <property=value> use value for given property
+ * -fs <local|namenode:port> specify a namenode
+ * -jt <local|jobtracker:port> specify a job tracker
+ * </pre></blockquote></p>
+ *
+ * <p>The general command line syntax is:</p>
+ * <p><tt><pre>
+ * bin/hadoop command [genericOptions] [commandOptions]
+ * </pre></tt></p>
+ *
+ * <p>Generic command line arguments <strong>might</strong> modify
+ * <code>Configuration </code> objects, given to constructors.</p>
+ *
+ * <p>The functionality is implemented using Commons CLI.</p>
+ *
+ * <p>Examples:</p>
* <p><blockquote><pre>
- * submit a job to local runner
- * </pre></blockquote>
+ * $ bin/hadoop dfs -fs darwin:8020 -ls /data
+ * list /data directory in dfs with namenode darwin:8020
+ *
+ * $ bin/hadoop dfs -D fs.default.name=darwin:8020 -ls /data
+ * list /data directory in dfs with namenode darwin:8020
+ *
+ * $ bin/hadoop dfs -conf hadoop-site.xml -ls /data
+ * list /data directory in dfs with conf specified in hadoop-site.xml
+ *
+ * $ bin/hadoop job -D mapred.job.tracker=darwin:50020 -submit job.xml
+ * submit a job to job tracker darwin:50020
+ *
+ * $ bin/hadoop job -jt darwin:50020 -submit job.xml
+ * submit a job to job tracker darwin:50020
+ *
+ * $ bin/hadoop job -jt local -submit job.xml
+ * submit a job to local runner
+ * </pre></blockquote></p>
*
* @see Tool
* @see ToolRunner
@@ -86,21 +90,27 @@
private CommandLine commandLine;
- /** Instantinates a GenericOptionsParser to parse only
- * the generic Hadoop arguments. The array of string arguments
- * other than the generic arguments can be obtained by
- * {@link #getRemainingArgs()}
- * @param conf the configuration to modify
- * @param args User-specified arguments
+ /**
+ * Create a <code>GenericOptionsParser<code> to parse only the generic Hadoop
+ * arguments.
+ *
+ * The array of string arguments other than the generic arguments can be
+ * obtained by {@link #getRemainingArgs()}.
+ *
+ * @param conf the <code>Configuration</code> to modify.
+ * @param args command-line arguments.
*/
public GenericOptionsParser(Configuration conf, String[] args) {
this(conf, new Options(), args);
}
/**
- * Instantinates a GenericOptionsParser to parse given options
- * as well as generic Hadoop options. The resulting <code>
- * CommandLine</code> object can be obtained by {@link #getCommandLine()}
+ * Create a <code>GenericOptionsParser</code> to parse given options as well
+ * as generic Hadoop options.
+ *
+ * The resulting <code>CommandLine</code> object can be obtained by
+ * {@link #getCommandLine()}.
+ *
* @param conf the configuration to modify
* @param options options built by the caller
* @param args User-specified arguments
@@ -110,9 +120,9 @@
}
/**
- * Returns an array of Strings containing only command-specific
- * arguments.
- * @return String array of remaining arguments not parsed
+ * Returns an array of Strings containing only application-specific arguments.
+ *
+ * @return array of <code>String</code>s containing the un-parsed arguments.
*/
public String[] getRemainingArgs() {
return commandLine.getArgs();
@@ -120,12 +130,14 @@
/**
* Returns the commons-cli <code>CommandLine</code> object
- * to process the parsed arguments. Note : if the object is
- * created with <code>GenericCommandLineParser(Configuration, String[])</code>,
- * then returned object will only contain parsed generic
- * options.
- * @return CommandLine object representing list of arguments
- * parsed against Options descriptor.
+ * to process the parsed arguments.
+ *
+ * Note: If the object is created with
+ * {@link #GenericOptionsParser(Configuration, String[])}, then returned
+ * object will only contain parsed generic options.
+ *
+ * @return <code>CommandLine</code> representing list of arguments
+ * parsed against Options descriptor.
*/
public CommandLine getCommandLine() {
return commandLine;
@@ -212,6 +224,11 @@
return args;
}
+ /**
+ * Print the usage message for generic command-line options supported.
+ *
+ * @param out stream to print the usage message to.
+ */
public static void printGenericCommandUsage(PrintStream out) {
out.println("Generic options supported are");
out.println("-conf <configuration file> specify an application configuration file");
Modified: lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Progressable.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Progressable.java?rev=588035&r1=588034&r2=588035&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Progressable.java (original)
+++ lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Progressable.java Wed Oct 24 14:24:51 2007
@@ -18,16 +18,18 @@
package org.apache.hadoop.util;
-import java.io.IOException;
-
-
-
/**
- * An interface for callbacks when an method makes some progress.
+ * A facility for reporting progress.
+ *
+ * <p>Clients and/or applications can use the provided <code>Progressable</code>
+ * to explicitly report progress to the Hadoop framework. This is especially
+ * important for operations which take an insignificant amount of time since,
+ * in-lieu of the reported progress, the framework has to assume that an error
+ * has occured and time-out the operation.</p>
*/
public interface Progressable {
- /** callback for reporting progress. Used by DFSclient to report
- * progress while writing a block of DFS file.
+ /**
+ * Report progress to the Hadoop framework.
*/
public void progress();
}
Modified: lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Tool.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Tool.java?rev=588035&r1=588034&r2=588035&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Tool.java (original)
+++ lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/Tool.java Wed Oct 24 14:24:51 2007
@@ -21,14 +21,58 @@
import org.apache.hadoop.conf.Configurable;
/**
- * A tool interface that support generic options handling
+ * A tool interface that supports handling of generic command-line options.
*
+ * <p><code>Tool</code>, is the standard for any Map-Reduce tool/application.
+ * The tool/application should delegate the handling of
+ * <a href="{@docRoot}/org/apache/hadoop/util/GenericOptionsParser.html#GenericOptions">
+ * standard command-line options</a> to {@link ToolRunner#run(Tool, String[])}
+ * and only handle its custom arguments.</p>
+ *
+ * <p>Here is how a typical <code>Tool</code> is implemented:</p>
+ * <p><blockquote><pre>
+ * public class MyApp extends Configured implements Tool {
+ *
+ * public int run(String[] args) throws Exception {
+ * // <code>Configuration</code> processed by <code>ToolRunner</code>
+ * Configuration conf = getConf();
+ *
+ * // Create a JobConf using the processed <code>conf</code>
+ * JobConf job = new JobConf(conf, MyApp.class);
+ *
+ * // Process custom command-line options
+ * Path in = new Path(args[1]);
+ * Path out = new Path(args[2]);
+ *
+ * // Specify various job-specific parameters
+ * job.setJobName("my-app");
+ * job.setInputPath(in);
+ * job.setOutputPath(out);
+ * job.setMapperClass(MyApp.MyMapper.class);
+ * job.setReducerClass(MyApp.MyReducer.class);
+ *
+ * // Submit the job, then poll for progress until the job is complete
+ * JobClient.runJob(job);
+ * }
+ *
+ * public static void main(String[] args) throws Exception {
+ * // Let <code>ToolRunner</code> handle generic command-line options
+ * int res = ToolRunner.run(new Configuration(), new Sort(), args);
+ *
+ * System.exit(res);
+ * }
+ * }
+ * </pre></blockquote></p>
+ *
+ * @see GenericOptionsParser
+ * @see ToolRunner
*/
public interface Tool extends Configurable {
/**
* Execute the command with the given arguments.
- * @param args command specific arguments
- * @return exit code
+ *
+ * @param args command specific arguments.
+ * @return exit code.
* @throws Exception
*/
int run(String [] args) throws Exception;
Modified: lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/ToolRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/ToolRunner.java?rev=588035&r1=588034&r2=588035&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/ToolRunner.java (original)
+++ lucene/hadoop/branches/branch-0.15/src/java/org/apache/hadoop/util/ToolRunner.java Wed Oct 24 14:24:51 2007
@@ -22,23 +22,34 @@
import org.apache.hadoop.conf.Configuration;
/**
- * ToolRunner can be used to run classes implementing {@link Tool}
- * interface. Static method {@link #run(Tool, String[])} is used.
- * {@link GenericOptionsParser} is used to parse the hadoop generic
- * arguments to modify the <code>Configuration</code>.
+ * A utility to help run {@link Tool}s.
+ *
+ * <p><code>ToolRunner</code> can be used to run classes implementing
+ * <code>Tool</code> interface. It works in conjunction with
+ * {@link GenericOptionsParser} to parse the
+ * <a href="{@docRoot}/org/apache/hadoop/util/GenericOptionsParser.html#GenericOptions">
+ * generic hadoop command line arguments</a> and modifies the
+ * <code>Configuration</code> of the <code>Tool</code>. The
+ * application-specific options are passed along without being modified.
+ * </p>
+ *
+ * @see Tool
+ * @see GenericOptionsParser
*/
public class ToolRunner {
/**
- * Runs the given Tool by {@link Tool#run(String[])}, with the
- * given arguments. Uses the given configuration, or builds one if null.
- * Sets the possibly modified version of the conf by Tool#setConf()
+ * Runs the given <code>Tool</code> by {@link Tool#run(String[])}, after
+ * parsing with the given generic arguments. Uses the given
+ * <code>Configuration</code>, or builds one if null.
*
- * @param conf Configuration object to use
- * @param tool The Tool to run
- * @param args the arguments to the tool(including generic arguments
- * , see {@link GenericOptionsParser})
- * @return exit code of the {@link Tool#run(String[])} method
+ * Sets the <code>Tool</code>'s configuration with the possibly modified
+ * version of the <code>conf</code>.
+ *
+ * @param conf <code>Configuration</code> for the <code>Tool</code>.
+ * @param tool <code>Tool</code> to run.
+ * @param args command-line arguments to the tool.
+ * @return exit code of the {@link Tool#run(String[])} method.
*/
public static int run(Configuration conf, Tool tool, String[] args)
throws Exception{
@@ -55,19 +66,24 @@
}
/**
- * Runs the tool with the tool's Configuration
+ * Runs the <code>Tool</code> with its <code>Configuration</code>.
+ *
* Equivalent to <code>run(tool.getConf(), tool, args)</code>.
- * @param tool The Tool to run
- * @param args the arguments to the tool(including generic arguments
- * , see {@link GenericOptionsParser})
- * @return exit code of the {@link Tool#run(String[])} method
+ *
+ * @param tool <code>Tool</code> to run.
+ * @param args command-line arguments to the tool.
+ * @return exit code of the {@link Tool#run(String[])} method.
*/
public static int run(Tool tool, String[] args)
throws Exception{
return run(tool.getConf(), tool, args);
}
- /** Delegates to GenericOptionsParser#printGenericCommandUsage() */
+ /**
+ * Prints generic command-line argurments and usage information.
+ *
+ * @param out stream to write usage information to.
+ */
public static void printGenericCommandUsage(PrintStream out) {
GenericOptionsParser.printGenericCommandUsage(out);
}
Modified: lucene/hadoop/branches/branch-0.15/src/test/org/apache/hadoop/mapred/NotificationTestCase.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.15/src/test/org/apache/hadoop/mapred/NotificationTestCase.java?rev=588035&r1=588034&r2=588035&view=diff
==============================================================================
--- lucene/hadoop/branches/branch-0.15/src/test/org/apache/hadoop/mapred/NotificationTestCase.java (original)
+++ lucene/hadoop/branches/branch-0.15/src/test/org/apache/hadoop/mapred/NotificationTestCase.java Wed Oct 24 14:24:51 2007
@@ -142,7 +142,7 @@
protected JobConf createJobConf() {
JobConf conf = super.createJobConf();
- conf.set("job.end.notification.url", getNotificationUrlTemplate());
+ conf.setJobEndNotificationURI(getNotificationUrlTemplate());
conf.setInt("job.end.retry.attempts", 3);
conf.setInt("job.end.retry.interval", 200);
return conf;