You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2013/09/17 00:11:07 UTC
git commit: TEZ-454. Fix Partitioner instantiation (part of TEZ-398).
(sseth)
Updated Branches:
refs/heads/TEZ-398 55a849332 -> c5ce1aba0
TEZ-454. Fix Partitioner instantiation (part of TEZ-398). (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/c5ce1aba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/c5ce1aba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/c5ce1aba
Branch: refs/heads/TEZ-398
Commit: c5ce1aba082a1c981da17e5d7d0de5f9f28e9c10
Parents: 55a8493
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Sep 16 15:10:45 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Sep 16 15:10:45 2013 -0700
----------------------------------------------------------------------
.../org/apache/tez/engine/common/TezEngineUtils.java | 15 +++++++++++++--
.../hadoop/MultiStageMRConfToTezTranslator.java | 4 ++++
2 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c5ce1aba/tez-engine/src/main/java/org/apache/tez/engine/common/TezEngineUtils.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/TezEngineUtils.java b/tez-engine/src/main/java/org/apache/tez/engine/common/TezEngineUtils.java
index d9bb280..f34092c 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/TezEngineUtils.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/TezEngineUtils.java
@@ -23,12 +23,16 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.engine.api.Partitioner;
public class TezEngineUtils {
+ private static final Log LOG = LogFactory
+ .getLog(TezEngineUtils.class);
+
public static String getTaskIdentifier(String vertexName, int taskIndex) {
return String.format("%s_%06d", vertexName, taskIndex);
}
@@ -48,8 +52,15 @@ public class TezEngineUtils {
@SuppressWarnings("unchecked")
public static Partitioner instantiatePartitioner(Configuration conf)
throws IOException {
- Class<? extends Partitioner> clazz = (Class<? extends Partitioner>) conf
- .getClass(TezJobConfig.TEZ_ENGINE_PARTITIONER_CLASS, Partitioner.class);
+ Class<? extends Partitioner> clazz;
+ try {
+ clazz = (Class<? extends Partitioner>) conf
+ .getClassByName(conf.get(TezJobConfig.TEZ_ENGINE_PARTITIONER_CLASS));
+ } catch (ClassNotFoundException e) {
+ throw new IOException("Unable to find Partitioner class in config", e);
+ }
+
+ LOG.info("ZZZ: Using partitioner class: " + clazz.getName());
Partitioner partitioner = null;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c5ce1aba/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
index 2d37a1c..6b68e95 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.mapreduce.hadoop.DeprecatedKeys.MultiStageKeys;
+import org.apache.tez.mapreduce.newpartition.MRPartitioner;
import com.google.common.base.Preconditions;
@@ -219,6 +220,9 @@ public class MultiStageMRConfToTezTranslator {
// Assuming no 0 map jobs, and the first stage is always a map.
int numStages = numIntermediateStages + (hasFinalReduceStage ? 2 : 1);
+ conf.set(TezJobConfig.TEZ_ENGINE_PARTITIONER_CLASS,
+ MRPartitioner.class.getName());
+
Configuration confs[] = new Configuration[numStages];
Configuration nonItermediateConf = MultiStageMRConfigUtil.extractStageConf(
conf, "");