You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/04/30 01:17:23 UTC
git commit: TEZ-1079. Make tez example jobs use the ToolRunner
framework (Devaraj K via bikas)
Repository: incubator-tez
Updated Branches:
refs/heads/master ec115a365 -> 9fa541ba2
TEZ-1079. Make tez example jobs use the ToolRunner framework (Devaraj K via bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/9fa541ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/9fa541ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/9fa541ba
Branch: refs/heads/master
Commit: 9fa541ba24235eec7dc7abc166bafd48ae0b39ed
Parents: ec115a3
Author: Bikas Saha <bi...@apache.org>
Authored: Tue Apr 29 16:17:20 2014 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue Apr 29 16:17:20 2014 -0700
----------------------------------------------------------------------
.../examples/BroadcastAndOneToOneExample.java | 14 ++++++--
.../mapreduce/examples/FilterLinesByWord.java | 34 +++++++++-----------
.../examples/FilterLinesByWordOneToOne.java | 32 +++++++++++-------
.../examples/GroupByOrderByMRRTest.java | 23 +++++++++----
.../mapreduce/examples/OrderedWordCount.java | 24 ++++++++++----
.../tez/mapreduce/examples/SecondarySort.java | 17 +++++++---
.../tez/mapreduce/examples/WordCount.java | 18 ++++++++---
7 files changed, 110 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fa541ba/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
index 22e11ba..5a972e8 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
@@ -31,6 +32,8 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.tez.client.AMConfiguration;
import org.apache.tez.client.TezClient;
@@ -64,7 +67,7 @@ import org.apache.tez.runtime.library.processor.SimpleProcessor;
import com.google.common.base.Preconditions;
-public class BroadcastAndOneToOneExample {
+public class BroadcastAndOneToOneExample extends Configured implements Tool {
public static class InputProcessor extends SimpleProcessor {
Text word = new Text();
@@ -243,10 +246,17 @@ public class BroadcastAndOneToOneExample {
tezSession.stop();
}
}
+
+ @Override
+ public int run(String[] args) throws Exception {
+ boolean status = run(getConf());
+ return status ? 0 : 1;
+ }
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
BroadcastAndOneToOneExample job = new BroadcastAndOneToOneExample();
- job.run(conf);
+ int status = ToolRunner.run(conf, job, args);
+ System.exit(status);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fa541ba/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index d976d17..dac35a7 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -30,6 +30,7 @@ import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -43,6 +44,8 @@ import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -85,14 +88,13 @@ import org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput;
import com.google.common.collect.Sets;
-public class FilterLinesByWord {
+public class FilterLinesByWord extends Configured implements Tool {
private static Log LOG = LogFactory.getLog(FilterLinesByWord.class);
public static final String FILTER_PARAM_NAME = "tez.runtime.examples.filterbyword.word";
private TezCounters counters = null;
- private int errorCode = 0;
private boolean exitOnCompletion = false;
public FilterLinesByWord(boolean exitOnCompletion) {
@@ -101,10 +103,12 @@ public class FilterLinesByWord {
private static void printUsage() {
System.err.println("Usage filtelinesrbyword <in> <out> <filter_word> [-generateSplitsInClient true/<false>]");
+ ToolRunner.printGenericCommandUsage(System.err);
}
- public void run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, TezException {
- Configuration conf = new Configuration();
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
Credentials credentials = new Credentials();
@@ -117,14 +121,12 @@ public class FilterLinesByWord {
} catch (ParseException e1) {
System.err.println("Invalid options");
printUsage();
- errorCode = 2;
- return;
+ return 2;
}
if (otherArgs.length != 3) {
printUsage();
- errorCode = 2;
- return;
+ return 2;
}
String inputPath = otherArgs[0];
@@ -134,8 +136,7 @@ public class FilterLinesByWord {
FileSystem fs = FileSystem.get(conf);
if (fs.exists(new Path(outputPath))) {
System.err.println("Output directory : " + outputPath + " already exists");
- errorCode = 2;
- return;
+ return 2;
}
TezConfiguration tezConf = new TezConfiguration(conf);
@@ -285,8 +286,7 @@ public class FilterLinesByWord {
dagStatus = dagClient.getDAGStatus(null);
} catch (TezException e) {
LOG.fatal("Failed to get application progress. Exiting");
- errorCode = -1;
- return;
+ return -1;
}
}
@@ -300,16 +300,14 @@ public class FilterLinesByWord {
ExampleDriver.printDAGStatus(dagClient, vNames, true, true);
LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
- errorCode = (dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1);
- return;
+ return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
}
- public static void main(String[] args) throws IOException, InterruptedException,
- ClassNotFoundException, TezException {
+ public static void main(String[] args) throws Exception {
FilterLinesByWord fl = new FilterLinesByWord(true);
- fl.run(args);
+ int status = ToolRunner.run(new Configuration(), fl, args);
if (fl.exitOnCompletion) {
- System.exit(fl.errorCode);
+ System.exit(status);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fa541ba/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
index 5d59d8c..bf765cb 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
@@ -18,7 +18,6 @@
package org.apache.tez.mapreduce.examples;
-import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
@@ -28,6 +27,7 @@ import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -37,6 +37,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -76,7 +78,7 @@ import org.apache.tez.runtime.api.TezRootInputInitializer;
import org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput;
import org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput;
-public class FilterLinesByWordOneToOne {
+public class FilterLinesByWordOneToOne extends Configured implements Tool {
private static Log LOG = LogFactory.getLog(FilterLinesByWordOneToOne.class);
@@ -85,12 +87,20 @@ public class FilterLinesByWordOneToOne {
private static void printUsage() {
System.err.println("Usage filterLinesByWordOneToOne <in> <out> <filter_word>"
+ " [-generateSplitsInClient true/<false>]");
+ ToolRunner.printGenericCommandUsage(System.err);
}
- public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException, TezException {
+ public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
- String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ String[] otherArgs = new GenericOptionsParser(conf, args)
+ .getRemainingArgs();
+ int status = ToolRunner.run(conf, new FilterLinesByWordOneToOne(),
+ otherArgs);
+ System.exit(status);
+ }
+ @Override
+ public int run(String[] otherArgs) throws Exception {
boolean generateSplitsInClient = false;
SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser();
try {
@@ -99,22 +109,23 @@ public class FilterLinesByWordOneToOne {
} catch (ParseException e1) {
System.err.println("Invalid options");
printUsage();
- System.exit(2);
+ return 2;
}
if (otherArgs.length != 3) {
printUsage();
- System.exit(2);
+ return 2;
}
String inputPath = otherArgs[0];
String outputPath = otherArgs[1];
String filterWord = otherArgs[2];
-
+
+ Configuration conf = getConf();
FileSystem fs = FileSystem.get(conf);
if (fs.exists(new Path(outputPath))) {
System.err.println("Output directory : " + outputPath + " already exists");
- System.exit(2);
+ return 2;
}
TezConfiguration tezConf = new TezConfiguration(conf);
@@ -261,7 +272,7 @@ public class FilterLinesByWordOneToOne {
dagStatus = dagClient.getDAGStatus(null);
} catch (TezException e) {
LOG.fatal("Failed to get application progress. Exiting");
- System.exit(-1);
+ return -1;
}
}
} finally {
@@ -271,7 +282,6 @@ public class FilterLinesByWordOneToOne {
ExampleDriver.printDAGStatus(dagClient, vNames);
LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
- System.exit(dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1);
+ return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fa541ba/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java
index 60ce3da..7875ef6 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java
@@ -24,6 +24,7 @@ import java.util.StringTokenizer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
@@ -35,6 +36,8 @@ import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.tez.client.TezClient;
import org.apache.tez.dag.api.TezConfiguration;
@@ -73,7 +76,7 @@ import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
* done
*
*/
-public class GroupByOrderByMRRTest {
+public class GroupByOrderByMRRTest extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(GroupByOrderByMRRTest.class);
@@ -146,8 +149,9 @@ public class GroupByOrderByMRRTest {
}
}
- public static void main(String[] args) throws Exception {
- Configuration conf = new Configuration();
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
// Configure intermediate reduces
conf.setInt(MRJobConfig.MRR_INTERMEDIATE_STAGES, 1);
@@ -168,7 +172,8 @@ public class GroupByOrderByMRRTest {
getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("Usage: groupbyorderbymrrtest <in> <out>");
- System.exit(2);
+ ToolRunner.printGenericCommandUsage(System.err);
+ return 2;
}
@SuppressWarnings("deprecation")
@@ -226,13 +231,19 @@ public class GroupByOrderByMRRTest {
dagStatus = dagClient.getDAGStatus(null);
} catch (TezException e) {
LOG.fatal("Failed to get application progress. Exiting");
- System.exit(-1);
+ return -1;
}
}
ExampleDriver.printDAGStatus(dagClient, vNames);
LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
- System.exit(dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1);
+ return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
}
+ public static void main(String[] args) throws Exception {
+ Configuration configuration = new Configuration();
+ GroupByOrderByMRRTest groupByOrderByMRRTest = new GroupByOrderByMRRTest();
+ int status = ToolRunner.run(configuration, groupByOrderByMRRTest, args);
+ System.exit(status);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fa541ba/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
index 9874f7b..78c2a37 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
@@ -32,6 +32,7 @@ import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
@@ -49,6 +50,8 @@ import org.apache.hadoop.mapreduce.split.TezGroupedSplitsInputFormat;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.tez.client.AMConfiguration;
@@ -97,7 +100,7 @@ import org.apache.tez.runtime.library.processor.SleepProcessor;
* Use -DINTER_JOB_SLEEP_INTERVAL=<N> where N is the sleep interval in seconds
* between the sequential DAGs.
*/
-public class OrderedWordCount {
+public class OrderedWordCount extends Configured implements Tool {
private static Log LOG = LogFactory.getLog(OrderedWordCount.class);
@@ -294,11 +297,13 @@ public class OrderedWordCount {
System.err.println("Usage: orderedwordcount <in> <out>" + options);
System.err.println("Usage (In Session Mode):"
+ " orderedwordcount <in1> <out1> ... <inN> <outN>" + options);
+ ToolRunner.printGenericCommandUsage(System.err);
}
- public static void main(String[] args) throws Exception {
- Configuration conf = new Configuration();
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
boolean generateSplitsInClient = false;
@@ -310,7 +315,7 @@ public class OrderedWordCount {
} catch (ParseException e1) {
System.err.println("Invalid options");
printUsage();
- System.exit(2);
+ return 2;
}
boolean useTezSession = conf.getBoolean("USE_TEZ_SESSION", true);
@@ -322,7 +327,7 @@ public class OrderedWordCount {
if (((otherArgs.length%2) != 0)
|| (!useTezSession && otherArgs.length != 2)) {
printUsage();
- System.exit(2);
+ return 2;
}
List<String> inputPaths = new ArrayList<String>();
@@ -497,7 +502,7 @@ public class OrderedWordCount {
dagStatus = dagClient.getDAGStatus(statusGetOpts);
} catch (TezException e) {
LOG.fatal("Failed to get application progress. Exiting");
- System.exit(-1);
+ return -1;
}
}
ExampleDriver.printDAGStatus(dagClient, vNames,
@@ -525,8 +530,9 @@ public class OrderedWordCount {
if (!useTezSession) {
ExampleDriver.printDAGStatus(dagClient, vNames);
LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
- System.exit(dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1);
+ return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
}
+ return 0;
}
private static void waitForTezSessionReady(TezSession tezSession)
@@ -548,4 +554,8 @@ public class OrderedWordCount {
}
}
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new OrderedWordCount(), args);
+ System.exit(res);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fa541ba/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java
index 35e48ac..cdae905 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
@@ -38,6 +39,8 @@ import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
/**
* This is an example Hadoop Map/Reduce application.
@@ -48,7 +51,7 @@ import org.apache.hadoop.util.GenericOptionsParser;
* To run: bin/hadoop jar build/hadoop-examples.jar secondarysort
* <i>in-dir</i> <i>out-dir</i>
*/
-public class SecondarySort {
+public class SecondarySort extends Configured implements Tool {
/**
* Define a pair of integers that are writable.
@@ -207,11 +210,13 @@ public class SecondarySort {
}
}
- public static void main(String[] args) throws Exception {
- Configuration conf = new Configuration();
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("Usage: secondarysort <in> <out>");
+ ToolRunner.printGenericCommandUsage(System.out);
System.exit(2);
}
Job job = new Job(conf, "secondary sort");
@@ -233,7 +238,11 @@ public class SecondarySort {
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
- System.exit(job.waitForCompletion(true) ? 0 : 1);
+ return job.waitForCompletion(true) ? 0 : 1;
}
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new SecondarySort(), args);
+ System.exit(res);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fa541ba/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
index 8a90259..3a6b8e2 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
@@ -23,6 +23,7 @@ import java.util.StringTokenizer;
import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
@@ -36,6 +37,8 @@ import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.tez.client.AMConfiguration;
@@ -71,7 +74,7 @@ import org.apache.tez.runtime.library.output.OnFileSortedOutput;
import com.google.common.base.Preconditions;
-public class WordCount {
+public class WordCount extends Configured implements Tool {
public static class TokenProcessor extends SimpleMRProcessor {
IntWritable one = new IntWritable(1);
Text word = new Text();
@@ -162,6 +165,7 @@ public class WordCount {
private static void printUsage() {
System.err.println("Usage: " + " wordcount <in1> <out1>");
+ ToolRunner.printGenericCommandUsage(System.err);
}
private Credentials credentials = new Credentials();
@@ -243,16 +247,22 @@ public class WordCount {
}
}
- public static void main(String[] args) throws Exception {
- Configuration conf = new Configuration();
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
printUsage();
- System.exit(2);
+ return 2;
}
WordCount job = new WordCount();
job.run(otherArgs[0], otherArgs[1], conf);
+ return 0;
}
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new WordCount(), args);
+ System.exit(res);
+ }
}