You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2013/09/25 09:31:09 UTC

[02/50] [abbrv] git commit: TEZ-470. Fix TestMRRJobsDAGApi to work with modified engine layer. (hitesh)

TEZ-470.  Fix TestMRRJobsDAGApi to work with modified engine layer. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/85a9d46e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/85a9d46e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/85a9d46e

Branch: refs/heads/master
Commit: 85a9d46e35756cacde115c64335268c6f55cbc3e
Parents: 6ca59ac
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Sep 20 14:43:13 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Sep 20 14:43:13 2013 -0700

----------------------------------------------------------------------
 .../apache/hadoop/mapred/YarnTezDagChild.java   |  3 ++
 .../apache/tez/mapreduce/TestMRRJobsDAGApi.java | 43 --------------------
 2 files changed, 3 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/85a9d46e/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 5034262..c749850 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -421,6 +421,9 @@ public class YarnTezDagChild {
               currentTask.run();
               currentTask.close();
             }
+            LOG.info("Task completed"
+                + ", taskAttemptId=" + currentTaskAttemptID
+                + ", fatalErrorOccurred=" + currentTask.hadFatalError());
             // TODONEWTEZ check if task had a fatal error before
             // sending completed event
             if (!currentTask.hadFatalError()) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/85a9d46e/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index 88dbd97..eb20876 100644
--- a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -300,8 +300,6 @@ public class TestMRRJobsDAGApi {
         .valueOf(new Random().nextInt(100000))));
     InputSplitInfo inputSplitInfo = MRHelpers.generateInputSplits(stage1Conf,
         remoteStagingDir);
-    InputSplitInfo inputSplitInfo1 = MRHelpers.generateInputSplits(stage1Conf,
-        remoteStagingDir);
 
     DAG dag = new DAG("testMRRSleepJobDagSubmit");
     Vertex stage1Vertex = new Vertex("map", new ProcessorDescriptor(
@@ -312,14 +310,6 @@ public class TestMRRJobsDAGApi {
         ReduceProcessor.class.getName()).setUserPayload(
         MRHelpers.createUserPayloadFromConf(stage2Conf)),
         1, Resource.newInstance(256, 1));
-    Vertex stage11Vertex = new Vertex("map1", new ProcessorDescriptor(
-        MapProcessor.class.getName()).setUserPayload(
-        MRHelpers.createUserPayloadFromConf(stage1Conf)),
-        inputSplitInfo1.getNumTasks(),  Resource.newInstance(256, 1));
-    Vertex stage22Vertex = new Vertex("ireduce1", new ProcessorDescriptor(
-        ReduceProcessor.class.getName()).setUserPayload(
-        MRHelpers.createUserPayloadFromConf(stage22Conf)),
-        2, Resource.newInstance(256, 1));
     Vertex stage3Vertex = new Vertex("reduce", new ProcessorDescriptor(
         ReduceProcessor.class.getName()).setUserPayload(
         MRHelpers.createUserPayloadFromConf(stage3Conf)),
@@ -353,44 +343,23 @@ public class TestMRRJobsDAGApi {
             LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
     stage1LocalResources.putAll(commonLocalResources);
 
-    Map<String, LocalResource> stage11LocalResources = new HashMap<String, LocalResource>();
-    stage11LocalResources.put(
-        inputSplitInfo1.getSplitsFile().getName(),
-        createLocalResource(remoteFs, inputSplitInfo1.getSplitsFile(),
-            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
-    stage11LocalResources.put(
-        inputSplitInfo1.getSplitsMetaInfoFile().getName(),
-        createLocalResource(remoteFs, inputSplitInfo1.getSplitsMetaInfoFile(),
-            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
-    stage11LocalResources.putAll(commonLocalResources);
-
     stage1Vertex.setJavaOpts(MRHelpers.getMapJavaOpts(stage1Conf));
     stage1Vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
     stage1Vertex.setTaskLocalResources(stage1LocalResources);
     stage1Vertex.setTaskEnvironment(commonEnv);
 
-    stage11Vertex.setJavaOpts(MRHelpers.getMapJavaOpts(stage1Conf));
-    stage11Vertex.setTaskLocationsHint(inputSplitInfo1.getTaskLocationHints());
-    stage11Vertex.setTaskLocalResources(stage11LocalResources);
-    stage11Vertex.setTaskEnvironment(commonEnv);
     // TODO env, resources
 
     stage2Vertex.setJavaOpts(MRHelpers.getReduceJavaOpts(stage2Conf));
     stage2Vertex.setTaskLocalResources(commonLocalResources);
     stage2Vertex.setTaskEnvironment(commonEnv);
 
-    stage22Vertex.setJavaOpts(MRHelpers.getReduceJavaOpts(stage22Conf));
-    stage22Vertex.setTaskLocalResources(commonLocalResources);
-    stage22Vertex.setTaskEnvironment(commonEnv);
-
     stage3Vertex.setJavaOpts(MRHelpers.getReduceJavaOpts(stage3Conf));
     stage3Vertex.setTaskLocalResources(commonLocalResources);
     stage3Vertex.setTaskEnvironment(commonEnv);
 
     dag.addVertex(stage1Vertex);
-    dag.addVertex(stage11Vertex);
     dag.addVertex(stage2Vertex);
-    dag.addVertex(stage22Vertex);
     dag.addVertex(stage3Vertex);
 
     Edge edge1 = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
@@ -398,26 +367,14 @@ public class TestMRRJobsDAGApi {
         SchedulingType.SEQUENTIAL, new OutputDescriptor(
         OnFileSortedOutput.class.getName()), new InputDescriptor(
                 ShuffledMergedInputLegacy.class.getName())));
-    Edge edge11 = new Edge(stage11Vertex, stage22Vertex, new EdgeProperty(
-        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, 
-        SchedulingType.SEQUENTIAL, new OutputDescriptor(
-        OnFileSortedOutput.class.getName()), new InputDescriptor(
-                ShuffledMergedInputLegacy.class.getName())));
     Edge edge2 = new Edge(stage2Vertex, stage3Vertex, new EdgeProperty(
         DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, 
         SchedulingType.SEQUENTIAL, new OutputDescriptor(
         OnFileSortedOutput.class.getName()), new InputDescriptor(
                 ShuffledMergedInputLegacy.class.getName())));
-    Edge edge3 = new Edge(stage22Vertex, stage3Vertex, new EdgeProperty(
-        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, 
-        SchedulingType.SEQUENTIAL, new OutputDescriptor(
-        OnFileSortedOutput.class.getName()), new InputDescriptor(
-                ShuffledMergedInputLegacy.class.getName())));
 
     dag.addEdge(edge1);
-    dag.addEdge(edge11);
     dag.addEdge(edge2);
-    dag.addEdge(edge3);
 
     Map<String, LocalResource> amLocalResources =
         new HashMap<String, LocalResource>();