You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2013/11/15 16:25:18 UTC
[1/2] git commit: TEZ-615. OrderedWordCount with
-DUSE_TEZ_SESSION=false throws NPE. (hitesh)
Updated Branches:
refs/heads/master d4bb730ea -> 727431fe7
TEZ-615. OrderedWordCount with -DUSE_TEZ_SESSION=false throws NPE. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/8a3dcf47
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/8a3dcf47
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/8a3dcf47
Branch: refs/heads/master
Commit: 8a3dcf4763ad1eaefd0b6d08685a3ade91977324
Parents: d4bb730
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Nov 15 07:24:26 2013 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Nov 15 07:24:26 2013 -0800
----------------------------------------------------------------------
.../java/org/apache/tez/client/TezClientUtils.java | 13 ++++++++++---
.../java/org/apache/tez/dag/app/rm/TaskScheduler.java | 2 +-
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8a3dcf47/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 68394b9..042ff91 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -364,9 +364,16 @@ public class TezClientUtils {
if(dag != null) {
for (Vertex v : dag.getVertices()) {
+ if (v.getTaskLocalResources() == null) {
+ v.setTaskLocalResources(new TreeMap<String, LocalResource>());
+ }
v.getTaskLocalResources().putAll(tezJarResources);
v.getTaskLocalResources().put(TezConfiguration.TEZ_PB_BINARY_CONF_NAME,
binaryConfLRsrc);
+
+ if (v.getTaskEnvironment() == null) {
+ v.setTaskEnvironment(new TreeMap<String, String>());
+ }
Map<String, String> taskEnv = v.getTaskEnvironment();
for (Map.Entry<String, String> entry : environment.entrySet()) {
String key = entry.getKey();
@@ -399,9 +406,9 @@ public class TezClientUtils {
}
localResources.put(TezConfiguration.TEZ_PB_PLAN_BINARY_NAME,
- TezClientUtils.createLocalResource(fs,
- binaryPath, LocalResourceType.FILE,
- LocalResourceVisibility.APPLICATION));
+ TezClientUtils.createLocalResource(fs,
+ binaryPath, LocalResourceType.FILE,
+ LocalResourceVisibility.APPLICATION));
if (Level.DEBUG.isGreaterOrEqual(Level.toLevel(amLogLevel))) {
Path textPath = localizeDagPlanAsText(dagPB, fs,
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8a3dcf47/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
index b2d3ffb..6eedb15 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
@@ -670,7 +670,7 @@ public class TaskScheduler extends AbstractService
boolean safeToRelease = true;
Priority topPendingPriority = amRmClient.getTopPriority();
Priority containerPriority = heldContainer.container.getPriority();
- if (topPendingPriority != null &&
+ if (isNew && topPendingPriority != null &&
containerPriority.compareTo(topPendingPriority) < 0) {
// this container is of lower priority and given to us by the RM for
// a task that will be matched after the current top priority. Keep
[2/2] git commit: TEZ-616. OrderedWordCount with
-generateSplitsInClient true throws NPE. (hitesh)
Posted by hi...@apache.org.
TEZ-616. OrderedWordCount with -generateSplitsInClient true throws NPE. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/727431fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/727431fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/727431fe
Branch: refs/heads/master
Commit: 727431fe7b0051bcd1b2f3856fa95ef4192daa43
Parents: 8a3dcf4
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Nov 15 07:24:57 2013 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Nov 15 07:24:57 2013 -0800
----------------------------------------------------------------------
.../org/apache/tez/mapreduce/examples/OrderedWordCount.java | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/727431fe/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
index 3ffd076..190678f 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
@@ -20,12 +20,14 @@ package org.apache.tez.mapreduce.examples;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
+import java.util.TreeMap;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
@@ -383,8 +385,11 @@ public class OrderedWordCount {
+ ", inputPath=" + inputPath
+ ", outputPath=" + outputPath);
- DAG dag = createDAG(fs, conf, null, stagingDir,
- dagIndex, inputPath, outputPath, generateSplitsInClient);
+ Map<String, LocalResource> localResources =
+ Collections.emptyMap();
+ DAG dag = createDAG(fs, conf, localResources,
+ stagingDir, dagIndex, inputPath, outputPath,
+ generateSplitsInClient);
if (useTezSession) {
LOG.info("Waiting for TezSession to get into ready state");