You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2013/09/25 09:31:09 UTC
[02/50] [abbrv] git commit: TEZ-470. Fix TestMRRJobsDAGApi to work
with modified engine layer. (hitesh)
TEZ-470. Fix TestMRRJobsDAGApi to work with modified engine layer. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/85a9d46e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/85a9d46e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/85a9d46e
Branch: refs/heads/master
Commit: 85a9d46e35756cacde115c64335268c6f55cbc3e
Parents: 6ca59ac
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Sep 20 14:43:13 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Sep 20 14:43:13 2013 -0700
----------------------------------------------------------------------
.../apache/hadoop/mapred/YarnTezDagChild.java | 3 ++
.../apache/tez/mapreduce/TestMRRJobsDAGApi.java | 43 --------------------
2 files changed, 3 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/85a9d46e/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 5034262..c749850 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -421,6 +421,9 @@ public class YarnTezDagChild {
currentTask.run();
currentTask.close();
}
+ LOG.info("Task completed"
+ + ", taskAttemptId=" + currentTaskAttemptID
+ + ", fatalErrorOccurred=" + currentTask.hadFatalError());
// TODONEWTEZ check if task had a fatal error before
// sending completed event
if (!currentTask.hadFatalError()) {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/85a9d46e/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index 88dbd97..eb20876 100644
--- a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -300,8 +300,6 @@ public class TestMRRJobsDAGApi {
.valueOf(new Random().nextInt(100000))));
InputSplitInfo inputSplitInfo = MRHelpers.generateInputSplits(stage1Conf,
remoteStagingDir);
- InputSplitInfo inputSplitInfo1 = MRHelpers.generateInputSplits(stage1Conf,
- remoteStagingDir);
DAG dag = new DAG("testMRRSleepJobDagSubmit");
Vertex stage1Vertex = new Vertex("map", new ProcessorDescriptor(
@@ -312,14 +310,6 @@ public class TestMRRJobsDAGApi {
ReduceProcessor.class.getName()).setUserPayload(
MRHelpers.createUserPayloadFromConf(stage2Conf)),
1, Resource.newInstance(256, 1));
- Vertex stage11Vertex = new Vertex("map1", new ProcessorDescriptor(
- MapProcessor.class.getName()).setUserPayload(
- MRHelpers.createUserPayloadFromConf(stage1Conf)),
- inputSplitInfo1.getNumTasks(), Resource.newInstance(256, 1));
- Vertex stage22Vertex = new Vertex("ireduce1", new ProcessorDescriptor(
- ReduceProcessor.class.getName()).setUserPayload(
- MRHelpers.createUserPayloadFromConf(stage22Conf)),
- 2, Resource.newInstance(256, 1));
Vertex stage3Vertex = new Vertex("reduce", new ProcessorDescriptor(
ReduceProcessor.class.getName()).setUserPayload(
MRHelpers.createUserPayloadFromConf(stage3Conf)),
@@ -353,44 +343,23 @@ public class TestMRRJobsDAGApi {
LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
stage1LocalResources.putAll(commonLocalResources);
- Map<String, LocalResource> stage11LocalResources = new HashMap<String, LocalResource>();
- stage11LocalResources.put(
- inputSplitInfo1.getSplitsFile().getName(),
- createLocalResource(remoteFs, inputSplitInfo1.getSplitsFile(),
- LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
- stage11LocalResources.put(
- inputSplitInfo1.getSplitsMetaInfoFile().getName(),
- createLocalResource(remoteFs, inputSplitInfo1.getSplitsMetaInfoFile(),
- LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
- stage11LocalResources.putAll(commonLocalResources);
-
stage1Vertex.setJavaOpts(MRHelpers.getMapJavaOpts(stage1Conf));
stage1Vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
stage1Vertex.setTaskLocalResources(stage1LocalResources);
stage1Vertex.setTaskEnvironment(commonEnv);
- stage11Vertex.setJavaOpts(MRHelpers.getMapJavaOpts(stage1Conf));
- stage11Vertex.setTaskLocationsHint(inputSplitInfo1.getTaskLocationHints());
- stage11Vertex.setTaskLocalResources(stage11LocalResources);
- stage11Vertex.setTaskEnvironment(commonEnv);
// TODO env, resources
stage2Vertex.setJavaOpts(MRHelpers.getReduceJavaOpts(stage2Conf));
stage2Vertex.setTaskLocalResources(commonLocalResources);
stage2Vertex.setTaskEnvironment(commonEnv);
- stage22Vertex.setJavaOpts(MRHelpers.getReduceJavaOpts(stage22Conf));
- stage22Vertex.setTaskLocalResources(commonLocalResources);
- stage22Vertex.setTaskEnvironment(commonEnv);
-
stage3Vertex.setJavaOpts(MRHelpers.getReduceJavaOpts(stage3Conf));
stage3Vertex.setTaskLocalResources(commonLocalResources);
stage3Vertex.setTaskEnvironment(commonEnv);
dag.addVertex(stage1Vertex);
- dag.addVertex(stage11Vertex);
dag.addVertex(stage2Vertex);
- dag.addVertex(stage22Vertex);
dag.addVertex(stage3Vertex);
Edge edge1 = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
@@ -398,26 +367,14 @@ public class TestMRRJobsDAGApi {
SchedulingType.SEQUENTIAL, new OutputDescriptor(
OnFileSortedOutput.class.getName()), new InputDescriptor(
ShuffledMergedInputLegacy.class.getName())));
- Edge edge11 = new Edge(stage11Vertex, stage22Vertex, new EdgeProperty(
- DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL, new OutputDescriptor(
- OnFileSortedOutput.class.getName()), new InputDescriptor(
- ShuffledMergedInputLegacy.class.getName())));
Edge edge2 = new Edge(stage2Vertex, stage3Vertex, new EdgeProperty(
DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL, new OutputDescriptor(
OnFileSortedOutput.class.getName()), new InputDescriptor(
ShuffledMergedInputLegacy.class.getName())));
- Edge edge3 = new Edge(stage22Vertex, stage3Vertex, new EdgeProperty(
- DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL, new OutputDescriptor(
- OnFileSortedOutput.class.getName()), new InputDescriptor(
- ShuffledMergedInputLegacy.class.getName())));
dag.addEdge(edge1);
- dag.addEdge(edge11);
dag.addEdge(edge2);
- dag.addEdge(edge3);
Map<String, LocalResource> amLocalResources =
new HashMap<String, LocalResource>();