You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/10/14 19:57:24 UTC
git commit: TEZ-1579. MR examples should be setting
mapreduce.framework.name to yarn-tez. Contributed by Krisztian Horvath.
Repository: tez
Updated Branches:
refs/heads/master 0ad4a4b88 -> 038ba9b6a
TEZ-1579. MR examples should be setting mapreduce.framework.name to
yarn-tez. Contributed by Krisztian Horvath.
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/038ba9b6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/038ba9b6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/038ba9b6
Branch: refs/heads/master
Commit: 038ba9b6ab1a8b5ccd4e3f5f13fc5f5413fea9f1
Parents: 0ad4a4b
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Oct 14 10:57:00 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Oct 14 10:57:00 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/mapreduce/examples/ExampleDriver.java | 10 +++++++++-
.../main/java/org/apache/tez/mapreduce/examples/Join.java | 6 ++++--
.../apache/tez/mapreduce/examples/MapredWordCount.java | 7 ++++---
.../apache/tez/mapreduce/examples/RandomTextWriter.java | 4 +++-
.../org/apache/tez/mapreduce/examples/RandomWriter.java | 4 +++-
.../main/java/org/apache/tez/mapreduce/examples/Sort.java | 4 +++-
7 files changed, 27 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/038ba9b6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7b6fbf7..2392e31 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ Release 0.6.0: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-1579. MR examples should be setting mapreduce.framework.name to yarn-tez.
Release 0.5.2: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/038ba9b6/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
index 3824607..e72fc87 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
@@ -23,6 +23,7 @@ import java.text.DecimalFormat;
import java.util.EnumSet;
import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ProgramDriver;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.TezException;
@@ -31,6 +32,7 @@ import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.Progress;
import org.apache.tez.dag.api.client.StatusGetOpts;
import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.mapreduce.hadoop.MRConfig;
/**
* A description of an example program based on its class and a
@@ -88,6 +90,12 @@ public class ExampleDriver {
System.exit(exitCode);
}
+ public static Configuration getTezDecoratedConfiguration() {
+ Configuration configuration = new Configuration();
+ configuration.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_TEZ_FRAMEWORK_NAME);
+ return configuration;
+ }
+
public static void printDAGStatus(DAGClient dagClient, String[] vertexNames)
throws IOException, TezException {
printDAGStatus(dagClient, vertexNames, false, false);
@@ -150,4 +158,4 @@ public class ExampleDriver {
}
}
-
+
http://git-wip-us.apache.org/repos/asf/tez/blob/038ba9b6/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java
index a3a65ec..8811232 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java
@@ -18,6 +18,8 @@
package org.apache.tez.mapreduce.examples;
+import static org.apache.tez.mapreduce.examples.ExampleDriver.getTezDecoratedConfiguration;
+
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -74,7 +76,7 @@ public class Join extends Configured implements Tool {
/**
* The main driver for sort program.
* Invoke this method to submit the map/reduce job.
- * @throws IOException When there is communication problems with the
+ * @throws Exception When there is communication problems with the
* job tracker.
*/
@SuppressWarnings("unchecked")
@@ -169,7 +171,7 @@ public class Join extends Configured implements Tool {
}
public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new Join(), args);
+ int res = ToolRunner.run(getTezDecoratedConfiguration(), new Join(), args);
System.exit(res);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/038ba9b6/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
index eef6223..91a502a 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
@@ -18,6 +18,8 @@
package org.apache.tez.mapreduce.examples;
+import static org.apache.tez.mapreduce.examples.ExampleDriver.getTezDecoratedConfiguration;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
@@ -26,7 +28,6 @@ import java.util.StringTokenizer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
@@ -155,8 +156,8 @@ public class MapredWordCount extends Configured implements Tool {
}
public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(),
- new MapredWordCount(), args);
+ int res = ToolRunner.run(getTezDecoratedConfiguration(),
+ new MapredWordCount(), args);
System.exit(res);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/038ba9b6/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
index 8251b78..8858d76 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
@@ -18,6 +18,8 @@
package org.apache.tez.mapreduce.examples;
+import static org.apache.tez.mapreduce.examples.ExampleDriver.getTezDecoratedConfiguration;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
@@ -245,7 +247,7 @@ public class RandomTextWriter extends Configured implements Tool {
}
public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new RandomTextWriter(), args);
+ int res = ToolRunner.run(getTezDecoratedConfiguration(), new RandomTextWriter(), args);
System.exit(res);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/038ba9b6/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java
index c9b51c7..a44c2c3 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java
@@ -18,6 +18,8 @@
package org.apache.tez.mapreduce.examples;
+import static org.apache.tez.mapreduce.examples.ExampleDriver.getTezDecoratedConfiguration;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
@@ -291,7 +293,7 @@ public class RandomWriter extends Configured implements Tool {
}
public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new RandomWriter(), args);
+ int res = ToolRunner.run(getTezDecoratedConfiguration(), new RandomWriter(), args);
System.exit(res);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/038ba9b6/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java
index 13e1bbc..6e1909e 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java
@@ -18,6 +18,8 @@
package org.apache.tez.mapreduce.examples;
+import static org.apache.tez.mapreduce.examples.ExampleDriver.getTezDecoratedConfiguration;
+
import java.net.URI;
import java.util.ArrayList;
import java.util.Date;
@@ -189,7 +191,7 @@ public class Sort<K,V> extends Configured implements Tool {
public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new Sort(), args);
+ int res = ToolRunner.run(getTezDecoratedConfiguration(), new Sort(), args);
System.exit(res);
}