You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2013/05/23 22:40:59 UTC
git commit: TEZ-150. Fix usage of mapreduce API keys for num maps and
num reduces. Removes Tez configuration for in-degree and out-degree. (sseth)
Updated Branches:
refs/heads/TEZ-1 268f3ac97 -> 4c2732c99
TEZ-150. Fix usage of mapreduce API keys for num maps and num reduces.
Removes Tez configuration for in-degree and out-degree. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/4c2732c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/4c2732c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/4c2732c9
Branch: refs/heads/TEZ-1
Commit: 4c2732c991a4cc76011acedbf9c17726c06bc145
Parents: 268f3ac
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu May 23 13:40:22 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu May 23 13:40:22 2013 -0700
----------------------------------------------------------------------
.../java/org/apache/tez/common/TezJobConfig.java | 16 +---------
.../org/apache/hadoop/mapred/LocalJobRunner.java | 1 -
.../tez/mapreduce/hadoop/DeprecatedKeys.java | 17 ++++------
.../processor/reduce/ReduceProcessor.java | 1 -
.../apache/tez/mapreduce/task/MRRuntimeTask.java | 4 ++
.../tez/mapreduce/hadoop/TestDeprecatedKeys.java | 23 +-------------
6 files changed, 14 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
index 0d6f6be..c0b0ae6 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -66,7 +66,7 @@ public class TezJobConfig {
*/
public static final String LOCAL_DIRS = "tez.engine.local.dirs";
public static final String DEFAULT_LOCAL_DIRS = "/tmp";
-
+
/**
* One local dir for the speicfic job.
*/
@@ -80,20 +80,6 @@ public class TezJobConfig {
public static final String DEFAULT_TASK_LOCAL_RESOURCE_DIR = "/tmp";
public static final String TEZ_TASK_WORKING_DIR = "tez.engine.task.working.dir";
-
- /**
- *
- */
- public static final String TEZ_ENGINE_TASK_INDEGREE =
- "tez.engine.task.in-degree";
- public static final int DEFAULT_TEZ_ENGINE_TASK_INDEGREE = 1;
-
- /**
- *
- */
- public static final String TEZ_ENGINE_TASK_OUTDEGREE =
- "tez.engine.task.out-degree";
- public static final int DEFAULT_TEZ_ENGINE_TASK_OUTDEGREE = 1;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 5c181e2..5dade2b 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -445,7 +445,6 @@ public class LocalJobRunner implements ClientProtocol {
JobConf localConf = new JobConf(job);
localConf.setUser(user);
localConf.set("mapreduce.jobtracker.address", "local");
- localConf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, mapIds.size());
setupChildMapredLocalDirs(reduceId, user, localConf);
// FIXME invalid task context
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index 01d4fe0..499159f 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -136,8 +136,6 @@ public class DeprecatedKeys {
registerMRToEngineKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD_BYTES, TezJobConfig.TEZ_ENGINE_IFILE_READAHEAD_BYTES);
registerMRToEngineKeyTranslation(MRJobConfig.RECORDS_BEFORE_PROGRESS, TezJobConfig.RECORDS_BEFORE_PROGRESS);
-
- registerMRToEngineKeyTranslation(MRJobConfig.JOB_LOCAL_DIR, MRConfig.LOCAL_DIR);
registerMRToEngineKeyTranslation(MRJobConfig.IO_SORT_FACTOR, TezJobConfig.TEZ_ENGINE_IO_SORT_FACTOR);
@@ -149,6 +147,12 @@ public class DeprecatedKeys {
registerMRToEngineKeyTranslation(MRJobConfig.MAP_COMBINE_MIN_SPILLS, TezJobConfig.TEZ_ENGINE_COMBINE_MIN_SPILLS);
+ // Counter replacement will work in this manner, as long as TezCounters
+ // extends MRCounters and is used directly by the Mapper/Reducer.
+ // When these counters are eventually translated over to MRCounters, this
+ // may break.
+ // Framework counters, like FILESYSTEM will likely be incompatible since
+ // they enum key belongs to a different package.
registerMRToEngineKeyTranslation(MRJobConfig.COUNTERS_MAX_KEY, TezJobConfig.COUNTERS_MAX_KEY);
registerMRToEngineKeyTranslation(MRJobConfig.COUNTER_GROUP_NAME_MAX_KEY, TezJobConfig.COUNTER_GROUP_NAME_MAX_KEY);
@@ -180,7 +184,7 @@ public class DeprecatedKeys {
registerMRToEngineKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, TezJobConfig.TEZ_ENGINE_SHUFFLE_ENABLE_MEMTOMEM);
registerMRToEngineKeyTranslation(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, TezJobConfig.TEZ_ENGINE_INPUT_BUFFER_PERCENT);
-
+
registerMRToEngineKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezJobConfig.DAG_CREDENTIALS_BINARY);
registerMRToEngineKeyTranslation("map.sort.class", TezJobConfig.TEZ_ENGINE_INTERNAL_SORTER_CLASS);
@@ -188,17 +192,10 @@ public class DeprecatedKeys {
registerMRToEngineKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_ENGINE_GROUP_COMPARATOR_CLASS);
registerMRToEngineKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_ENGINE_INTERMEDIATE_INPUT_KEY_SECONDARY_COMPARATOR_CLASS);
-
- registerMRToEngineKeyTranslation(MRJobConfig.NUM_REDUCES, TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE);
- registerMRToEngineKeyTranslation(MRJobConfig.NUM_MAPS, TezJobConfig.TEZ_ENGINE_TASK_INDEGREE);
}
private static void addDeprecatedKeys() {
- // Adding deprecation for num_maps, num_reduces - in case some component in
- // Hadoop MR land uses these. Tez components already use tez-in/out-degree.
- _(MRJobConfig.NUM_REDUCES, TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE);
- _(MRJobConfig.NUM_MAPS, TezJobConfig.TEZ_ENGINE_TASK_INDEGREE);
}
private static void registerMRToEngineKeyTranslation(String mrKey,
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
index ce49726..9a6fb5b 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
@@ -338,7 +338,6 @@ implements Processor {
throws IOException, InterruptedException {
super.localizeConfiguration(jobConf);
jobConf.setBoolean(JobContext.TASK_ISMAP, false);
- jobConf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, numMapTasks);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index 34aaa1b..a3d5684 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -170,6 +170,10 @@ public class MRRuntimeTask extends RuntimeTask {
job.set(MRJobConfig.JOB_LOCAL_DIR, job.get(TezJobConfig.JOB_LOCAL_DIR));
job.set(MRConfig.LOCAL_DIR, job.get(TezJobConfig.LOCAL_DIRS));
+ if (job.get(TezJobConfig.DAG_CREDENTIALS_BINARY) != null) {
+ job.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY,
+ job.get(TezJobConfig.DAG_CREDENTIALS_BINARY));
+ }
// setup the child's attempt directories
// Do the task-type specific localization
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
index 16e783e..99aad74 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
@@ -18,28 +18,9 @@
package org.apache.tez.mapreduce.hadoop;
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.tez.common.TezJobConfig;
-import org.junit.Test;
public class TestDeprecatedKeys {
- @Test
- public void testNumTasksDeprecation() {
-
- Configuration conf = new Configuration(false);
- conf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, 6);
- conf.setInt(TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE, 3);
-
-
- DeprecatedKeys.init();
- JobConf jobConf = new JobConf(conf);
-
- Assert.assertEquals(6, jobConf.getNumMapTasks());
- Assert.assertEquals(3, jobConf.getNumReduceTasks());
- }
-
+ // TODO Add tests
+
}