You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2013/06/04 07:33:03 UTC

[01/50] [abbrv] git commit: TEZ-121: build break due to MAPREDUCE-4067 (protocol exception types)

Updated Branches:
  refs/heads/master d6cd24bbe -> 809bbd964


TEZ-121: build break due to MAPREDUCE-4067 (protocol exception types)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/97e1fa96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/97e1fa96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/97e1fa96

Branch: refs/heads/master
Commit: 97e1fa9671b3f2b04c5d2ef7547bb882da351ea7
Parents: c86c279
Author: mikelid <mi...@microsoft.com>
Authored: Mon May 13 11:16:09 2013 -0700
Committer: mikelid <mi...@microsoft.com>
Committed: Mon May 13 11:16:09 2013 -0700

----------------------------------------------------------------------
 .../org/apache/tez/mapreduce/NotRunningJob.java    |   30 +++++++-------
 1 files changed, 15 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/97e1fa96/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
index 7199edf..370c8f0 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
@@ -18,6 +18,7 @@
 
 package org.apache.tez.mapreduce;
 
+import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -65,7 +66,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -101,7 +101,7 @@ public class NotRunningJob implements MRClientProtocol {
 
   @Override
   public FailTaskAttemptResponse failTaskAttempt(
-      FailTaskAttemptRequest request) throws YarnRemoteException {
+      FailTaskAttemptRequest request) throws IOException {
     FailTaskAttemptResponse resp =
       recordFactory.newRecordInstance(FailTaskAttemptResponse.class);
     return resp;
@@ -109,7 +109,7 @@ public class NotRunningJob implements MRClientProtocol {
 
   @Override
   public GetCountersResponse getCounters(GetCountersRequest request)
-      throws YarnRemoteException {
+      throws IOException {
     GetCountersResponse resp =
       recordFactory.newRecordInstance(GetCountersResponse.class);
     Counters counters = recordFactory.newRecordInstance(Counters.class);
@@ -120,7 +120,7 @@ public class NotRunningJob implements MRClientProtocol {
 
   @Override
   public GetDiagnosticsResponse getDiagnostics(GetDiagnosticsRequest request)
-      throws YarnRemoteException {
+      throws IOException {
     GetDiagnosticsResponse resp =
       recordFactory.newRecordInstance(GetDiagnosticsResponse.class);
     resp.addDiagnostics("");
@@ -129,7 +129,7 @@ public class NotRunningJob implements MRClientProtocol {
 
   @Override
   public GetJobReportResponse getJobReport(GetJobReportRequest request)
-      throws YarnRemoteException {
+      throws IOException {
     JobReport jobReport =
       recordFactory.newRecordInstance(JobReport.class);
     jobReport.setJobId(request.getJobId());
@@ -150,7 +150,7 @@ public class NotRunningJob implements MRClientProtocol {
   @Override
   public GetTaskAttemptCompletionEventsResponse getTaskAttemptCompletionEvents(
       GetTaskAttemptCompletionEventsRequest request)
-      throws YarnRemoteException {
+      throws IOException {
     GetTaskAttemptCompletionEventsResponse resp =
       recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class);
     resp.addAllCompletionEvents(new ArrayList<TaskAttemptCompletionEvent>());
@@ -159,14 +159,14 @@ public class NotRunningJob implements MRClientProtocol {
 
   @Override
   public GetTaskAttemptReportResponse getTaskAttemptReport(
-      GetTaskAttemptReportRequest request) throws YarnRemoteException {
+      GetTaskAttemptReportRequest request) throws IOException {
     //not invoked by anybody
     throw new NotImplementedException();
   }
 
   @Override
   public GetTaskReportResponse getTaskReport(GetTaskReportRequest request)
-      throws YarnRemoteException {
+      throws IOException {
     GetTaskReportResponse resp =
       recordFactory.newRecordInstance(GetTaskReportResponse.class);
     TaskReport report = recordFactory.newRecordInstance(TaskReport.class);
@@ -181,7 +181,7 @@ public class NotRunningJob implements MRClientProtocol {
 
   @Override
   public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request)
-      throws YarnRemoteException {
+      throws IOException {
     GetTaskReportsResponse resp =
       recordFactory.newRecordInstance(GetTaskReportsResponse.class);
     resp.addAllTaskReports(new ArrayList<TaskReport>());
@@ -190,7 +190,7 @@ public class NotRunningJob implements MRClientProtocol {
 
   @Override
   public KillJobResponse killJob(KillJobRequest request)
-      throws YarnRemoteException {
+      throws IOException {
     KillJobResponse resp =
       recordFactory.newRecordInstance(KillJobResponse.class);
     return resp;
@@ -198,7 +198,7 @@ public class NotRunningJob implements MRClientProtocol {
 
   @Override
   public KillTaskResponse killTask(KillTaskRequest request)
-      throws YarnRemoteException {
+      throws IOException {
     KillTaskResponse resp =
       recordFactory.newRecordInstance(KillTaskResponse.class);
     return resp;
@@ -206,7 +206,7 @@ public class NotRunningJob implements MRClientProtocol {
 
   @Override
   public KillTaskAttemptResponse killTaskAttempt(
-      KillTaskAttemptRequest request) throws YarnRemoteException {
+      KillTaskAttemptRequest request) throws IOException {
     KillTaskAttemptResponse resp =
       recordFactory.newRecordInstance(KillTaskAttemptResponse.class);
     return resp;
@@ -214,21 +214,21 @@ public class NotRunningJob implements MRClientProtocol {
 
   @Override
   public GetDelegationTokenResponse getDelegationToken(
-      GetDelegationTokenRequest request) throws YarnRemoteException {
+      GetDelegationTokenRequest request) throws IOException {
     /* Should not be invoked by anyone. */
     throw new NotImplementedException();
   }
 
   @Override
   public RenewDelegationTokenResponse renewDelegationToken(
-      RenewDelegationTokenRequest request) throws YarnRemoteException {
+      RenewDelegationTokenRequest request) throws IOException {
     /* Should not be invoked by anyone. */
     throw new NotImplementedException();
   }
 
   @Override
   public CancelDelegationTokenResponse cancelDelegationToken(
-      CancelDelegationTokenRequest request) throws YarnRemoteException {
+      CancelDelegationTokenRequest request) throws IOException {
     /* Should not be invoked by anyone. */
     throw new NotImplementedException();
   }


[40/50] [abbrv] git commit: TEZ-174. Fix hadoop-maven-plugin version should be managed through pluginManagement

Posted by ss...@apache.org.
TEZ-174. Fix hadoop-maven-plugin version should be managed through pluginManagement


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/d3b19212
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/d3b19212
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/d3b19212

Branch: refs/heads/master
Commit: d3b19212c7cd01aa26c31ad0eb40d50662144f54
Parents: a679189
Author: Giridharan Kesavan <gk...@apache.org>
Authored: Fri May 31 06:18:45 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri May 31 07:15:53 2013 -0700

----------------------------------------------------------------------
 pom.xml             |    5 +++++
 tez-dag-api/pom.xml |    1 -
 2 files changed, 5 insertions(+), 1 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d3b19212/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0e4f8e7..721df2f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -301,6 +301,11 @@
           <version>2.3.1</version>
         </plugin>
         <plugin>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-maven-plugins</artifactId>
+          <version>${hadoop.version}</version>
+        </plugin>
+        <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-assembly-plugin</artifactId>
           <version>2.4</version>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d3b19212/tez-dag-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag-api/pom.xml b/tez-dag-api/pom.xml
index 3cb0952..2a91b4d 100644
--- a/tez-dag-api/pom.xml
+++ b/tez-dag-api/pom.xml
@@ -64,7 +64,6 @@
       <plugin>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-maven-plugins</artifactId>
-        <version>3.0.0-SNAPSHOT</version>
         <executions>
           <execution>
             <id>compile-protoc</id>


[10/50] [abbrv] git commit: TEZ-128. Add additional unit tests for TaskAttempt.

Posted by ss...@apache.org.
TEZ-128. Add additional unit tests for TaskAttempt.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/d660948d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/d660948d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/d660948d

Branch: refs/heads/master
Commit: d660948d7eaa3536a3bcd97e16524aa626b4ad7c
Parents: 4c89e15
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue May 14 15:31:07 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue May 14 15:31:07 2013 -0700

----------------------------------------------------------------------
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java      |    5 -
 .../tez/dag/app/dag/impl/TestTaskAttempt.java      |  409 ++++++++++++---
 .../apache/tez/dag/app/dag/impl/TestTaskImpl.java  |    2 +
 3 files changed, 325 insertions(+), 91 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d660948d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 2564ffa..264b3ad 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -116,7 +116,6 @@ public class TaskAttemptImpl implements TaskAttempt,
   protected EventHandler eventHandler;
   private final TezTaskAttemptID attemptId;
   private final Clock clock;
-//  private final TaskAttemptListener taskAttemptListener;
   private final List<String> diagnostics = new ArrayList<String>();
   private final Lock readLock;
   private final Lock writeLock;
@@ -923,10 +922,6 @@ public class TaskAttemptImpl implements TaskAttempt,
       // Send out events to the Task - indicating TaskAttemptTermination(F/K)
       ta.sendEvent(new TaskEventTAUpdate(ta.attemptId, helper
           .getTaskEventType()));
-
-      if (event instanceof DiagnosableEvent) {
-        ta.addDiagnosticInfo(((DiagnosableEvent) event).getDiagnosticInfo());
-      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d660948d/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index a18bd3b..369bced 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -20,13 +20,18 @@ package org.apache.tez.dag.app.dag.impl;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.fs.FileStatus;
@@ -61,10 +66,14 @@ import org.apache.tez.dag.app.dag.event.DAGEvent;
 import org.apache.tez.dag.app.dag.event.DAGEventType;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEvent;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminating;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventKillRequest;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
+import org.apache.tez.dag.app.dag.event.TaskEventTAUpdate;
+import org.apache.tez.dag.app.rm.AMSchedulerEventTAEnded;
+import org.apache.tez.dag.app.rm.AMSchedulerEventTALaunchRequest;
 import org.apache.tez.dag.app.rm.container.AMContainerMap;
 import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -72,6 +81,7 @@ import org.apache.tez.dag.records.TezTaskID;
 import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
 import org.junit.Test;
+import org.mockito.ArgumentCaptor;
 
 @SuppressWarnings({ "unchecked", "rawtypes" })
 public class TestTaskAttempt {
@@ -102,88 +112,96 @@ public class TestTaskAttempt {
   // testMRAppHistory(app);
   // }
 
-  // @Test
-  // // Verifies that the launch request is based on the hosts.
-  // // TODO Move to the client.
-  // // TODO Add a test that verifies that the LocationHint is used as it should
-  // be.
-  // public void testSingleRackRequest() throws Exception {
-  // TaskAttemptImpl.ScheduleTaskattemptTransition sta =
-  // new TaskAttemptImpl.ScheduleTaskattemptTransition();
-  //
-  // EventHandler eventHandler = mock(EventHandler.class);
-  // String[] hosts = new String[3];
-  // hosts[0] = "host1";
-  // hosts[1] = "host2";
-  // hosts[2] = "host3";
-  // TaskSplitMetaInfo splitInfo = new TaskSplitMetaInfo(hosts, 0,
-  // 128 * 1024 * 1024l);
-  //
-  // TaskAttemptImpl mockTaskAttempt = createMapTaskAttemptImpl2ForTest(
-  // eventHandler, splitInfo);
-  // TaskAttemptEventSchedule mockTAEvent =
-  // mock(TaskAttemptEventSchedule.class);
-  // doReturn(false).when(mockTAEvent).isRescheduled();
-  //
-  // sta.transition(mockTaskAttempt, mockTAEvent);
-  //
-  // ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
-  // verify(eventHandler, times(2)).handle(arg.capture());
-  // if (!(arg.getAllValues().get(1) instanceof
-  // AMSchedulerTALaunchRequestEvent)) {
-  // Assert.fail("Second Event not of type ContainerRequestEvent");
-  // }
-  // AMSchedulerTALaunchRequestEvent tlrE = (AMSchedulerTALaunchRequestEvent)
-  // arg
-  // .getAllValues().get(1);
-  // String[] requestedRacks = tlrE.getRacks();
-  // // Only a single occurrence of /DefaultRack
-  // assertEquals(1, requestedRacks.length);
-  // }
+  @Test
+  public void testLocalityRequest() {
 
-  // @Test
-  // // Tests that an attempt is made to resolve the localized hosts to racks.
-  // // TODO Move to the client.
-  // public void testHostResolveAttempt() throws Exception {
-  // TaskAttemptImpl.ScheduleTaskattemptTransition sta =
-  // new TaskAttemptImpl.ScheduleTaskattemptTransition();
-  //
-  // EventHandler eventHandler = mock(EventHandler.class);
-  // String hosts[] = new String[] {"192.168.1.1", "host2", "host3"};
-  // String resolved[] = new String[] {"host1", "host2", "host3"};
-  // TaskSplitMetaInfo splitInfo =
-  // new TaskSplitMetaInfo(hosts, 0, 128 * 1024 * 1024l);
-  //
-  // TaskAttemptImpl mockTaskAttempt =
-  // createMapTaskAttemptImpl2ForTest(eventHandler, splitInfo);
-  // TaskAttemptImpl spyTa = spy(mockTaskAttempt);
-  // when(spyTa.resolveHosts(hosts)).thenReturn(resolved);
-  //
-  // TaskAttemptEventSchedule mockTAEvent =
-  // mock(TaskAttemptEventSchedule.class);
-  // doReturn(false).when(mockTAEvent).isRescheduled();
-  //
-  // sta.transition(spyTa, mockTAEvent);
-  // verify(spyTa).resolveHosts(hosts);
-  // ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
-  // verify(eventHandler, times(2)).handle(arg.capture());
-  // if (!(arg.getAllValues().get(1) instanceof
-  // AMSchedulerTALaunchRequestEvent)) {
-  // Assert.fail("Second Event not of type ContainerRequestEvent");
-  // }
-  // Map<String, Boolean> expected = new HashMap<String, Boolean>();
-  // expected.put("host1", true);
-  // expected.put("host2", true);
-  // expected.put("host3", true);
-  // AMSchedulerTALaunchRequestEvent cre =
-  // (AMSchedulerTALaunchRequestEvent) arg.getAllValues().get(1);
-  // String[] requestedHosts = cre.getHosts();
-  // for (String h : requestedHosts) {
-  // expected.remove(h);
-  // }
-  // assertEquals(0, expected.size());
-  // }
-  //
+    TaskAttemptImpl.ScheduleTaskattemptTransition sta =
+        new TaskAttemptImpl.ScheduleTaskattemptTransition();
+
+    EventHandler eventHandler = mock(EventHandler.class);
+    String[] hosts = new String[3];
+    hosts[0] = "host1";
+    hosts[1] = "host2";
+    hosts[2] = "host3";
+    TaskLocationHint locationHint = new TaskLocationHint(hosts, null);
+
+    TezTaskID taskID = new TezTaskID(
+        new TezVertexID(new TezDAGID("1", 1, 1), 1), 1);
+    TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
+        mock(TaskAttemptListener.class), 1, new TezConfiguration(),
+        mock(Token.class), new Credentials(), new SystemClock(),
+        mock(TaskHeartbeatHandler.class), mock(AppContext.class),
+        MAP_PROCESSOR_NAME, locationHint, BuilderUtils.newResource(1024, 1),
+        new HashMap<String, LocalResource>(), new HashMap<String, String>(),
+        "", false);
+
+    TaskAttemptEventSchedule sEvent = mock(TaskAttemptEventSchedule.class);
+
+    sta.transition(taImpl, sEvent);
+
+    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+    verify(eventHandler, times(1)).handle(arg.capture());
+    if (!(arg.getAllValues().get(0) instanceof AMSchedulerEventTALaunchRequest)) {
+      fail("Second event not of type "
+          + AMSchedulerEventTALaunchRequest.class.getName());
+    }
+    // TODO Move the Rack request check to the client after TEZ-125 is fixed.
+    AMSchedulerEventTALaunchRequest lre = (AMSchedulerEventTALaunchRequest) arg
+        .getAllValues().get(0);
+    String[] requestedRacks = lre.getRacks();
+    assertEquals(1, requestedRacks.length);
+    assertEquals(3, lre.getHosts().length);
+    for (int i = 0; i < 3; i++) {
+      assertEquals("host" + (i + 1), lre.getHosts()[i]);
+    }
+  }
+
+
+  @Test
+  // Tests that an attempt is made to resolve the localized hosts to racks.
+  // TODO Move to the client post TEZ-125.
+  public void testHostResolveAttempt() throws Exception {
+    TaskAttemptImpl.ScheduleTaskattemptTransition sta =
+        new TaskAttemptImpl.ScheduleTaskattemptTransition();
+
+    EventHandler eventHandler = mock(EventHandler.class);
+    String hosts[] = new String[] { "192.168.1.1", "host2", "host3" };
+    String resolved[] = new String[] { "host1", "host2", "host3" };
+    TaskLocationHint locationHint = new TaskLocationHint(hosts, null);
+
+    TezTaskID taskID = new TezTaskID(
+        new TezVertexID(new TezDAGID("1", 1, 1), 1), 1);
+    TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
+        mock(TaskAttemptListener.class), 1, new TezConfiguration(),
+        mock(Token.class), new Credentials(), new SystemClock(),
+        mock(TaskHeartbeatHandler.class), mock(AppContext.class),
+        MAP_PROCESSOR_NAME, locationHint, BuilderUtils.newResource(1024, 1),
+        new HashMap<String, LocalResource>(), new HashMap<String, String>(),
+        "", false);
+    TaskAttemptImpl spyTa = spy(taImpl);
+    when(spyTa.resolveHosts(hosts)).thenReturn(resolved);
+
+    TaskAttemptEventSchedule mockTAEvent = mock(TaskAttemptEventSchedule.class);
+
+    sta.transition(spyTa, mockTAEvent);
+    verify(spyTa).resolveHosts(hosts);
+    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+    verify(eventHandler, times(1)).handle(arg.capture());
+    if (!(arg.getAllValues().get(0) instanceof AMSchedulerEventTALaunchRequest)) {
+      fail("Second Event not of type ContainerRequestEvent");
+    }
+    Map<String, Boolean> expected = new HashMap<String, Boolean>();
+    expected.put("host1", true);
+    expected.put("host2", true);
+    expected.put("host3", true);
+    AMSchedulerEventTALaunchRequest cre = (AMSchedulerEventTALaunchRequest) arg
+        .getAllValues().get(0);
+    String[] requestedHosts = cre.getHosts();
+    for (String h : requestedHosts) {
+      expected.remove(h);
+    }
+    assertEquals(0, expected.size());
+  }
 
   // @Test
   // // Verifies accounting of slot_milli counters. Time spent in running tasks.
@@ -325,10 +343,105 @@ public class TestTaskAttempt {
     // null));
     assertFalse(eventHandler.internalError);
   }
+  
+  @Test
+  // Ensure ContainerTerminating and ContainerTerminated is handled correctly by
+  // the TaskAttempt
+  public void testContainerTerminationWhileRunning() throws Exception {
+    ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
+    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+        appId, 0);
+    TezDAGID dagID = new TezDAGID(appId, 1);
+    TezVertexID vertexID = new TezVertexID(dagID, 1);
+    TezTaskID taskID = new TezTaskID(vertexID, 1);
+    TezTaskAttemptID taskAttemptID = new TezTaskAttemptID(taskID, 0);
 
-  // TODO Add a similar test for TERMINATING.
-  // Ensure ContainerTerminated is handled correctly by the TaskAttempt
+    MockEventHandler eventHandler = spy(new MockEventHandler());
+    TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+    when(taListener.getAddress()).thenReturn(
+        new InetSocketAddress("localhost", 0));
+
+    TezConfiguration tezConf = new TezConfiguration();
+    tezConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+    tezConf.setBoolean("fs.file.impl.disable.cache", true);
+
+    TaskLocationHint locationHint = new TaskLocationHint(
+        new String[] { "127.0.0.1" }, null);
+    Resource resource = BuilderUtils.newResource(1024, 1);
+    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+    Map<String, String> environment = new HashMap<String, String>();
+    String javaOpts = "";
+
+    NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
+    ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+    Container container = mock(Container.class);
+    when(container.getId()).thenReturn(contId);
+    when(container.getNodeId()).thenReturn(nid);
+    when(container.getNodeHttpAddress()).thenReturn("localhost:0");
+
+    AppContext appCtx = mock(AppContext.class);
+    AMContainerMap containers = new AMContainerMap(
+        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        appCtx);
+    containers.addContainerIfNew(container);
+
+    doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
+    doReturn(containers).when(appCtx).getAllContainers();
+
+    TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
+        taListener, 1, tezConf, mock(Token.class), new Credentials(),
+        new SystemClock(), mock(TaskHeartbeatHandler.class), appCtx,
+        MAP_PROCESSOR_NAME, locationHint, resource, localResources,
+        environment, javaOpts, false);
+
+    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+
+    taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, null));
+    // At state STARTING.
+    taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID, contId,
+        null, -1));
+    assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(),
+        TaskAttemptState.RUNNING);
+
+    int expectedEventsAtRunning = 3;
+    verify(eventHandler, times(expectedEventsAtRunning)).handle(arg.capture());
+
+    taImpl.handle(new TaskAttemptEventContainerTerminating(taskAttemptID,
+        "Terminating"));
+    assertFalse(
+        "InternalError occurred trying to handle TA_CONTAINER_TERMINATING",
+        eventHandler.internalError);
+    
+    assertEquals("Task attempt is not in the  FAILED state", taImpl.getState(),
+        TaskAttemptState.FAILED);
+
+    assertEquals(1, taImpl.getDiagnostics().size());
+    assertEquals("Terminating", taImpl.getDiagnostics().get(0));
+
+    int expectedEvenstAfterTerminating = expectedEventsAtRunning + 3;
+    arg = ArgumentCaptor.forClass(Event.class);
+    verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture());
+    
+    verifyEventType(
+        arg.getAllValues().subList(expectedEventsAtRunning,
+            expectedEvenstAfterTerminating), TaskEventTAUpdate.class, 1);
+    verifyEventType(
+        arg.getAllValues().subList(expectedEventsAtRunning,
+            expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1);
+    
+    taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID,
+        "Terminated"));
+    int expectedEventAfterTerminated = expectedEvenstAfterTerminating + 0;
+    arg = ArgumentCaptor.forClass(Event.class);
+    verify(eventHandler, times(expectedEventAfterTerminated)).handle(arg.capture());
+
+    assertEquals(2, taImpl.getDiagnostics().size());
+    assertEquals("Terminated", taImpl.getDiagnostics().get(1));
+  }
+
+  
   @Test
+  // Ensure ContainerTerminated is handled correctly by the TaskAttempt
   public void testContainerTerminatedWhileRunning() throws Exception {
     ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
     ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
@@ -382,11 +495,14 @@ public class TestTaskAttempt {
         null, -1));
     assertEquals("Task attempt is not in running state", taImpl.getState(),
         TaskAttemptState.RUNNING);
-    taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID, null));
+    taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID, "Terminated"));
     assertFalse(
         "InternalError occurred trying to handle TA_CONTAINER_TERMINATED",
         eventHandler.internalError);
-    // TODO Verify diagnostics
+    
+    assertEquals("Terminated", taImpl.getDiagnostics().get(0));
+    
+    // TODO Ensure TA_TERMINATING after this is ingored.
   }
 
   @Test
@@ -456,6 +572,98 @@ public class TestTaskAttempt {
   }
 
   @Test
+  // Ensure ContainerTerminating and ContainerTerminated is handled correctly by
+  // the TaskAttempt
+  public void testContainerTerminatedAfterSuccess() throws Exception {
+    ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
+    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+        appId, 0);
+    TezDAGID dagID = new TezDAGID(appId, 1);
+    TezVertexID vertexID = new TezVertexID(dagID, 1);
+    TezTaskID taskID = new TezTaskID(vertexID, 1);
+    TezTaskAttemptID taskAttemptID = new TezTaskAttemptID(taskID, 0);
+
+    MockEventHandler eventHandler = spy(new MockEventHandler());
+    TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+    when(taListener.getAddress()).thenReturn(
+        new InetSocketAddress("localhost", 0));
+
+    TezConfiguration tezConf = new TezConfiguration();
+    tezConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+    tezConf.setBoolean("fs.file.impl.disable.cache", true);
+
+    TaskLocationHint locationHint = new TaskLocationHint(
+        new String[] { "127.0.0.1" }, null);
+    Resource resource = BuilderUtils.newResource(1024, 1);
+    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+    Map<String, String> environment = new HashMap<String, String>();
+    String javaOpts = "";
+
+    NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
+    ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+    Container container = mock(Container.class);
+    when(container.getId()).thenReturn(contId);
+    when(container.getNodeId()).thenReturn(nid);
+    when(container.getNodeHttpAddress()).thenReturn("localhost:0");
+
+    AppContext appCtx = mock(AppContext.class);
+    AMContainerMap containers = new AMContainerMap(
+        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        appCtx);
+    containers.addContainerIfNew(container);
+
+    doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
+    doReturn(containers).when(appCtx).getAllContainers();
+
+    TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
+        taListener, 1, tezConf, mock(Token.class), new Credentials(),
+        new SystemClock(), mock(TaskHeartbeatHandler.class), appCtx,
+        MAP_PROCESSOR_NAME, locationHint, resource, localResources,
+        environment, javaOpts, false);
+
+    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+
+    taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, null));
+    // At state STARTING.
+    taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID, contId,
+        null, -1));
+    assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(),
+        TaskAttemptState.RUNNING);
+
+    int expectedEventsAtRunning = 3;
+    verify(eventHandler, times(expectedEventsAtRunning)).handle(arg.capture());
+
+    taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE));
+
+    assertEquals("Task attempt is not in the  SUCCEEDED state", taImpl.getState(),
+        TaskAttemptState.SUCCEEDED);
+
+    assertEquals(0, taImpl.getDiagnostics().size());
+
+    int expectedEvenstAfterTerminating = expectedEventsAtRunning + 3;
+    arg = ArgumentCaptor.forClass(Event.class);
+    verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture());
+    
+    verifyEventType(
+        arg.getAllValues().subList(expectedEventsAtRunning,
+            expectedEvenstAfterTerminating), TaskEventTAUpdate.class, 1);
+    verifyEventType(
+        arg.getAllValues().subList(expectedEventsAtRunning,
+            expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1);
+    
+    taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID,
+        "Terminated"));
+    int expectedEventAfterTerminated = expectedEvenstAfterTerminating + 0;
+    arg = ArgumentCaptor.forClass(Event.class);
+    verify(eventHandler, times(expectedEventAfterTerminated)).handle(arg.capture());
+
+    // Verify that the diagnostic message included in the Terminated event is not
+    // captured - TA already succeeded.
+    assertEquals(0, taImpl.getDiagnostics().size());
+  }
+
+  
+  @Test
   // Verifies that multiple TooManyFetchFailures are handled correctly by the
   // TaskAttempt.
   public void testMultipleTooManyFetchFailures() throws Exception {
@@ -467,7 +675,8 @@ public class TestTaskAttempt {
     TezTaskID taskID = new TezTaskID(vertexID, 1);
     TezTaskAttemptID taskAttemptID = new TezTaskAttemptID(taskID, 0);
 
-    MockEventHandler eventHandler = new MockEventHandler();
+    MockEventHandler mockEh = new MockEventHandler();
+    MockEventHandler eventHandler = spy(mockEh);
     TaskAttemptListener taListener = mock(TaskAttemptListener.class);
     when(taListener.getAddress()).thenReturn(
         new InetSocketAddress("localhost", 0));
@@ -513,8 +722,20 @@ public class TestTaskAttempt {
         TaskAttemptEventType.TA_DONE));
     assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
         TaskAttemptState.SUCCEEDED);
+    
+    int expectedEventsTillSucceeded = 6;
+    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+    verify(eventHandler, times(expectedEventsTillSucceeded)).handle(arg.capture());
+    verifyEventType(arg.getAllValues(), TaskEventTAUpdate.class, 2);
+    
     taImpl.handle(new TaskAttemptEvent(taskAttemptID,
         TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES));
+    int expectedEventsAfterFetchFailure = expectedEventsTillSucceeded + 3;
+    verify(eventHandler, times(expectedEventsAfterFetchFailure)).handle(arg.capture());
+    verifyEventType(
+        arg.getAllValues().subList(expectedEventsTillSucceeded,
+            expectedEventsAfterFetchFailure), TaskEventTAUpdate.class, 1);
+
     assertEquals("Task attempt is not in FAILED state", taImpl.getState(),
         TaskAttemptState.FAILED);
     taImpl.handle(new TaskAttemptEvent(taskAttemptID,
@@ -524,8 +745,24 @@ public class TestTaskAttempt {
     assertFalse(
         "InternalError occurred trying to handle TA_TOO_MANY_FETCH_FAILURES",
         eventHandler.internalError);
+    // No new events.
+    verify(eventHandler, times(expectedEventsAfterFetchFailure)).handle(
+        arg.capture());
   }
 
+  private void verifyEventType(List<Event> events,
+      Class<? extends Event> eventClass, int expectedOccurences) {
+    int count = 0;
+    for (Event e : events) {
+      if (eventClass.isInstance(e)) {
+        count++;
+      }
+    }
+    assertEquals(
+        "Mismatch in num occurences of event: " + eventClass.getCanonicalName(),
+        expectedOccurences, count);
+  }
+  
   public static class MockEventHandler implements EventHandler {
     public boolean internalError;
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d660948d/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 516ea5e..100f2aa 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -363,6 +363,8 @@ public class TestTaskImpl {
     assertTaskSucceededState();
 
   }
+  
+  // TODO Add test to validate the correct commit attempt.
 
   @SuppressWarnings("rawtypes")
   private class MockTaskImpl extends TaskImpl {


[04/50] [abbrv] git commit: TEZ-25. cleanup of unused import statements

Posted by ss...@apache.org.
TEZ-25. cleanup of unused import statements


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/45db1be3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/45db1be3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/45db1be3

Branch: refs/heads/master
Commit: 45db1be34f349dd7a2c94854d68d92a7e93a5719
Parents: cb5758b
Author: mikelid <mi...@microsoft.com>
Authored: Mon May 13 12:01:09 2013 -0700
Committer: mikelid <mi...@microsoft.com>
Committed: Mon May 13 12:01:09 2013 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/tez/dag/api/DAG.java  |    5 -----
 .../org/apache/tez/dag/api/DagTypeConverters.java  |    4 ----
 2 files changed, 0 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/45db1be3/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
index c18c720..efaf276 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -26,13 +26,8 @@ import java.util.Map.Entry;
 
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.tez.dag.api.*;
-import org.apache.tez.dag.api.DAGPlan.PlanKeyValuePair.Builder;
 import org.apache.tez.dag.api.DAGPlan.PlanTaskLocationHint;
-import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
-import org.apache.tez.dag.api.EdgeProperty.SourceType;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
 import org.apache.tez.dag.api.DAGPlan.*;
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/45db1be3/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index 04404ba..afbabad 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -17,15 +17,11 @@
  */
 package org.apache.tez.dag.api;
 
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;


[28/50] [abbrv] git commit: TEZ-156. Fix build break due to removal of BuilderUtils

Posted by ss...@apache.org.
TEZ-156. Fix build break due to removal of BuilderUtils


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/b5ebd492
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/b5ebd492
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/b5ebd492

Branch: refs/heads/master
Commit: b5ebd4923729dd3f0eb77339f2f1338265153588
Parents: 5d8adeb
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri May 24 10:13:42 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri May 24 10:13:42 2013 -0700

----------------------------------------------------------------------
 .../dag/app/rm/container/AMContainerHelpers.java   |   14 ++++++--------
 .../org/apache/tez/mapreduce/NotRunningJob.java    |    6 +++---
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |    7 +++----
 3 files changed, 12 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b5ebd492/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 0550895..6a9ae2e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -136,10 +136,9 @@ public class AMContainerHelpers {
     // Construct the actual Container
     // The null fields are per-container and will be constructed for each
     // container separately.
-    ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext(
-        appContext.getDAG().getUserName(), localResources,
-        environment, null, serviceData, taskCredentialsBuffer, applicationACLs);
-
+    ContainerLaunchContext container =
+        ContainerLaunchContext.newInstance(localResources, environment, null,
+            serviceData, taskCredentialsBuffer, applicationACLs);
     return container;
   }
 
@@ -188,10 +187,9 @@ public class AMContainerHelpers {
     }
 
     // Construct the actual Container
-    ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext(
-        commonContainerSpec.getUser(), lResources, myEnv, commands,
-        myServiceData, commonContainerSpec.getTokens().duplicate(),
-        acls);
+    ContainerLaunchContext container =
+        ContainerLaunchContext.newInstance(lResources, myEnv, commands,
+            myServiceData, commonContainerSpec.getTokens(), acls);
 
     return container;
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b5ebd492/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
index 370c8f0..ba7ab5a 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
@@ -87,9 +87,9 @@ public class NotRunningJob implements MRClientProtocol {
 
     // Setting AppState to NEW and finalStatus to UNDEFINED as they are never
     // used for a non running job
-    return BuilderUtils.newApplicationReport(unknownAppId, unknownAttemptId,
-        "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
-        "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f);
+    return ApplicationReport.newInstance(unknownAppId, unknownAttemptId, "N/A",
+        "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", "N/A",
+        0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, "TEZ_MRR");
   }
 
   NotRunningJob(ApplicationReport applicationReport, JobState jobState) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b5ebd492/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 9ece649..a79a4f9 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -816,10 +816,9 @@ public class YARNRunner implements ClientProtocol {
         = new HashMap<ApplicationAccessType, String>();
 
     // Setup ContainerLaunchContext for AM container
-    ContainerLaunchContext amContainer = BuilderUtils
-        .newContainerLaunchContext(UserGroupInformation
-            .getCurrentUser().getShortUserName(), localResources,
-            environment, vargsFinal, null, securityTokens, acls);
+    ContainerLaunchContext amContainer =
+        ContainerLaunchContext.newInstance(localResources, environment,
+            vargsFinal, null, securityTokens, acls);
 
     // Set up the ApplicationSubmissionContext
     ApplicationSubmissionContext appContext = Records


[16/50] [abbrv] git commit: TEZ-144. Fix ContainerLaunch to always use ContainerTokens, irrespective of security (YARN-617). (sseth)

Posted by ss...@apache.org.
TEZ-144. Fix ContainerLaunch to always use ContainerTokens, irrespective
of security (YARN-617). (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/0d5027d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/0d5027d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/0d5027d0

Branch: refs/heads/master
Commit: 0d5027d00d72292ce8acd45441d02b7d5e6e8ecb
Parents: de5ee03
Author: Siddharth Seth <ss...@apache.org>
Authored: Sat May 18 20:43:36 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Sat May 18 20:43:36 2013 -0700

----------------------------------------------------------------------
 .../dag/app/launcher/ContainerLauncherImpl.java    |   16 ++++++--------
 1 files changed, 7 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/0d5027d0/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index 48a41c4..f1a16b8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -360,15 +360,13 @@ public class ContainerLauncherImpl extends AbstractService implements
 
     final InetSocketAddress cmAddr =
         NetUtils.createSocketAddr(containerManagerBindAddr);
-    UserGroupInformation user = UserGroupInformation.getCurrentUser();
-
-    if (UserGroupInformation.isSecurityEnabled()) {
-      Token<ContainerTokenIdentifier> token =
-          ProtoUtils.convertFromProtoFormat(containerToken, cmAddr);
-      // the user in createRemoteUser in this context has to be ContainerID
-      user = UserGroupInformation.createRemoteUser(containerID.toString());
-      user.addToken(token);
-    }
+
+    // the user in createRemoteUser in this context has to be ContainerID
+    UserGroupInformation user = UserGroupInformation
+        .createRemoteUser(containerID.toString());
+    Token<ContainerTokenIdentifier> token = ProtoUtils.convertFromProtoFormat(
+        containerToken, cmAddr);
+    user.addToken(token);
 
     ContainerManager proxy = user
         .doAs(new PrivilegedAction<ContainerManager>() {


[25/50] [abbrv] git commit: TEZ-32. Fix logging support for MapReduce tasks and AM. (sseth)

Posted by ss...@apache.org.
TEZ-32. Fix logging support for MapReduce tasks and AM. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/268f3ac9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/268f3ac9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/268f3ac9

Branch: refs/heads/master
Commit: 268f3ac97b081294fcdc22f7258890f219285fca
Parents: 36198a6
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu May 23 11:17:43 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu May 23 11:17:43 2013 -0700

----------------------------------------------------------------------
 .../org/apache/tez/dag/api/TezConfiguration.java   |    4 +-
 .../apache/tez/dag/utils/TezEngineChildJVM.java    |    1 -
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |  113 ++++++++-------
 3 files changed, 61 insertions(+), 57 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/268f3ac9/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 4a407b3..3018ea1 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -45,7 +45,9 @@ public class TezConfiguration extends Configuration {
   // TODO Should not be required once all tokens are handled via AppSubmissionContext
   public static final String JOB_SUBMIT_DIR = TEZ_PREFIX + "jobSubmitDir";
   public static final String APPLICATION_TOKENS_FILE = "appTokens";
-  
+  public static final String DAG_APPLICATION_MASTER_CLASS = 
+      "org.apache.tez.dag.app.DAGAppMaster";
+
   public static final String DAG_AM_TASK_LISTENER_THREAD_COUNT = 
                                 TEZ_PREFIX + "task.listener.thread-count";
   public static final int DAG_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT = 30;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/268f3ac9/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
index 727ded5..bc92ed8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.app.rm.container.AMContainerHelpers;
 import org.apache.tez.dag.records.TezVertexID;
 
 public class TezEngineChildJVM {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/268f3ac9/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 7582a5e..9ece649 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -22,7 +22,6 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -45,8 +44,6 @@ import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.ProtocolSignature;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TaskLog;
-import org.apache.hadoop.mapred.TaskLog.LogName;
 import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
 import org.apache.hadoop.mapreduce.ClusterMetrics;
 import org.apache.hadoop.mapreduce.Counters;
@@ -70,7 +67,6 @@ import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIden
 import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
 import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader;
 import org.apache.hadoop.mapreduce.v2.LogParams;
-import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.security.Credentials;
@@ -93,11 +89,10 @@ import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
 import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.Edge;
@@ -120,8 +115,6 @@ public class YARNRunner implements ClientProtocol {
 
   private static final Log LOG = LogFactory.getLog(YARNRunner.class);
 
-  private final RecordFactory recordFactory =
-      RecordFactoryProvider.getRecordFactory(null);
   private ResourceMgrDelegate resMgrDelegate;
   private ClientCache clientCache;
   private Configuration conf;
@@ -172,6 +165,7 @@ public class YARNRunner implements ClientProtocol {
     }
   }
 
+  @VisibleForTesting
   @Private
   /**
    * Used for testing mostly.
@@ -210,12 +204,6 @@ public class YARNRunner implements ClientProtocol {
     return resMgrDelegate.getClusterMetrics();
   }
 
-  @VisibleForTesting
-  Token<?> getDelegationTokenFromHS(MRClientProtocol hsProxy)
-      throws IOException, InterruptedException {
-    throw new UnsupportedOperationException("No HistoryServer for Tez");
-  }
-
   @Override
   public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
       throws IOException, InterruptedException {
@@ -384,17 +372,6 @@ public class YARNRunner implements ClientProtocol {
         MRJobConfig.MAPRED_ADMIN_USER_ENV,
         MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV));
 
-    // FIXME is this really required?
-    // Add stdout/stderr env
-    environment.put(
-        MRJobConfig.STDOUT_LOGFILE_ENV,
-        getTaskLogFile(TaskLog.LogName.STDOUT)
-        );
-    environment.put(
-        MRJobConfig.STDERR_LOGFILE_ENV,
-        getTaskLogFile(TaskLog.LogName.STDERR)
-        );
-
   }
 
   private static String getChildEnv(Configuration jobConf, boolean isMap) {
@@ -418,19 +395,7 @@ public class YARNRunner implements ClientProtocol {
     }
   }
 
-  private static String getTaskLogFile(LogName filter) {
-    return ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR
-        + filter.toString();
-  }
-
-  private static void setupLog4jProperties(Configuration jobConf,
-      boolean isMap,
-      Vector<String> vargs,
-      long logSize) {
-    String logLevel = getChildLogLevel(jobConf, isMap);
-    MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
-  }
-
+  
   private void setupMapReduceEnv(Configuration jobConf,
       Map<String, String> environment, boolean isMap) throws IOException {
 
@@ -474,13 +439,10 @@ public class YARNRunner implements ClientProtocol {
     // FIXME: don't think this is also needed given we already set java
     // properties.
     // TODO Change this not to use JobConf.
-    long logSize = TaskLog.getTaskLogLength(new JobConf(jobConf));
-    Vector<String> logProps = new Vector<String>(4);
-    setupLog4jProperties(jobConf, isMap, logProps, logSize);
-    Iterator<String> it = logProps.iterator();
+    String log4jCmdLineProperties = getLog4jCmdLineProperties(jobConf, isMap);
     StringBuffer buffer = new StringBuffer();
-    while (it.hasNext()) {
-      buffer.append(" " + it.next());
+    if (log4jCmdLineProperties != null && log4jCmdLineProperties != "") {
+      buffer.append(" " + log4jCmdLineProperties);
     }
 
     // FIXME supposedly required for streaming, should we remove it and let
@@ -492,7 +454,7 @@ public class YARNRunner implements ClientProtocol {
       hadoopClientOpts = hadoopClientOpts + " ";
     }
     hadoopClientOpts = hadoopClientOpts + buffer.toString();
-    // environment.put("HADOOP_CLIENT_OPTS", hadoopClientOpts);
+    //environment.put("HADOOP_CLIENT_OPTS", hadoopClientOpts);
 
     // FIXME for this to work, we need YARN-561 and the task runtime changed
     // to use YARN-561
@@ -730,7 +692,7 @@ public class YARNRunner implements ClientProtocol {
     ApplicationId applicationId = resMgrDelegate.getApplicationId();
 
     // Setup resource requirements
-    Resource capability = recordFactory.newRecordInstance(Resource.class);
+    Resource capability = Records.newRecord(Resource.class);
     capability.setMemory(
         conf.getInt(TezConfiguration.DAG_AM_RESOURCE_MEMORY_MB,
             TezConfiguration.DEFAULT_DAG_AM_RESOURCE_MEMORY_MB));
@@ -757,9 +719,10 @@ public class YARNRunner implements ClientProtocol {
 //     LOG.error(" !!!!!!!!!");
 //     vargs.add("-Xdebug -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=8002,server=y,suspend=y");
     
-    // FIXME set up logging related properties
     // TODO -Dtez.root.logger??
-    // MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
+    String amLogLevel = jobConf.get(MRJobConfig.MR_AM_LOG_LEVEL,
+        MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
+    addLog4jSystemProperties(amLogLevel, vargs);
 
     // FIXME admin command opts and user command opts for tez?
     String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
@@ -775,7 +738,7 @@ public class YARNRunner implements ClientProtocol {
         MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
     vargs.add(mrAppMasterUserOptions);
 
-    vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS);
+    vargs.add(TezConfiguration.DAG_APPLICATION_MASTER_CLASS);
     vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
         Path.SEPARATOR + ApplicationConstants.STDOUT);
     vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
@@ -859,8 +822,9 @@ public class YARNRunner implements ClientProtocol {
             environment, vargsFinal, null, securityTokens, acls);
 
     // Set up the ApplicationSubmissionContext
-    ApplicationSubmissionContext appContext =
-        recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
+    ApplicationSubmissionContext appContext = Records
+        .newRecord(ApplicationSubmissionContext.class);
+    
     appContext.setApplicationId(applicationId);                // ApplicationId
     appContext.setResource(capability);                        // resource
     appContext.setQueue(                                       // Queue name
@@ -975,7 +939,7 @@ public class YARNRunner implements ClientProtocol {
 
   private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type)
       throws IOException {
-    LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
+    LocalResource rsrc = Records.newRecord(LocalResource.class);
     FileStatus rsrcStat = fs.getFileStatus(p);
     rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
         .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
@@ -1127,6 +1091,7 @@ public class YARNRunner implements ClientProtocol {
     }
   }
   
+  @SuppressWarnings("deprecation")
   private String getMapJavaOpts(Configuration jobConf) {
     // follows pattern from YARN MapReduceChildJVM.java
     String adminOpts = "";
@@ -1142,9 +1107,11 @@ public class YARNRunner implements ClientProtocol {
                     JobConf.MAPRED_TASK_JAVA_OPTS,
                     JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
 
-    return adminOpts + " " + userOpts;
+    return adminOpts.trim() + " " + userOpts.trim() + " "
+    + getLog4jCmdLineProperties(jobConf, true);
   }
   
+  @SuppressWarnings("deprecation")
   private String getReduceJavaOpts(Configuration jobConf) {
     // follows pattern from YARN MapReduceChildJVM.java 
     String adminOpts = "";
@@ -1159,6 +1126,42 @@ public class YARNRunner implements ClientProtocol {
                 jobConf.get(
                     JobConf.MAPRED_TASK_JAVA_OPTS,
                     JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
-    return adminOpts + " " + userOpts;
+    
+    return adminOpts.trim() + " " + userOpts.trim() + " "
+        + getLog4jCmdLineProperties(jobConf, false);
   }
-}
+
+  private static String getLog4jCmdLineProperties(Configuration jobConf,
+      boolean isMap) {
+    Vector<String> logProps = new Vector<String>(4);
+    addLog4jSystemProperties(getChildLogLevel(jobConf, isMap), logProps);
+    StringBuilder sb = new StringBuilder();
+    for (String str : logProps) {
+      sb.append(str).append(" ");
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Add the JVM system properties necessary to configure
+   * {@link ContainerLogAppender}.
+   * 
+   * @param logLevel
+   *          the desired log level (eg INFO/WARN/DEBUG)
+   * @param vargs
+   *          the argument list to append to
+   */
+  private static void addLog4jSystemProperties(String logLevel,
+      List<String> vargs) {
+    vargs.add("-Dlog4j.configuration=container-log4j.properties");
+    // TODO Fix the remaining properties after YARN-720 is fixed.
+    // May need to introduce a log4j properties file for tez tasks.
+    vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "="
+        + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    // Setting this to 0 to avoid log size restrictions.
+    // Should be enforced by YARN.
+    vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + 0);
+    vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
+  }
+
+}
\ No newline at end of file


[19/50] [abbrv] git commit: TEZ-149. MRRuntimeTask should read local dir configuration from config rather than from env. (sseth)

Posted by ss...@apache.org.
TEZ-149. MRRuntimeTask should read local dir configuration from config
rather than from env. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/9fb21703
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/9fb21703
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/9fb21703

Branch: refs/heads/master
Commit: 9fb21703c8bd73a5ffd15a39d7a598bcc0ce32c7
Parents: 8298190
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue May 21 11:28:06 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue May 21 11:28:06 2013 -0700

----------------------------------------------------------------------
 .../common/sort/impl/dflt/DefaultSorter.java       |    2 +-
 .../processor/reduce/ReduceProcessor.java          |    2 +-
 .../apache/tez/mapreduce/task/MRRuntimeTask.java   |    2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fb21703/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
index 2cf5ab4..e567492 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
@@ -219,7 +219,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
     }
     if (partition < 0 || partition >= partitions) {
       throw new IOException("Illegal partition for " + key + " (" +
-          partition + ")");
+          partition + ")" + ", TotalPartitions: " + partitions);
     }
     checkSpillException();
     bufferRemaining -= METASIZE;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fb21703/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
index 9ae07ee..ce49726 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
@@ -303,7 +303,7 @@ implements Processor {
     org.apache.hadoop.mapreduce.Reducer reducer =
         (org.apache.hadoop.mapreduce.Reducer)
         ReflectionUtils.newInstance(taskContext.getReducerClass(), job);
-    
+
     org.apache.hadoop.mapreduce.RecordWriter trackedRW = 
         new org.apache.hadoop.mapreduce.RecordWriter() {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fb21703/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index 6091d4b..3b51889 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -198,7 +198,7 @@ public class MRRuntimeTask extends RuntimeTask {
   private static void setupDistributedCacheConfig(final JobConf job)
       throws IOException {
 
-    String localWorkDir = System.getenv(System.getenv(Environment.PWD.name()));
+    String localWorkDir = (job.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
     // ^ ^ all symlinks are created in the current work-dir
 
     // Update the configuration object with localized archives.


[09/50] [abbrv] git commit: TEZ-127. YARN-615 build break fix (bikas)

Posted by ss...@apache.org.
TEZ-127. YARN-615 build break fix (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/4c89e15c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/4c89e15c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/4c89e15c

Branch: refs/heads/master
Commit: 4c89e15c61787742326ee943ba17ab61b65427fc
Parents: 8193b5f
Author: Bikas Saha <bi...@apache.org>
Authored: Tue May 14 14:34:15 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue May 14 14:34:15 2013 -0700

----------------------------------------------------------------------
 .../dag/app/rm/container/AMContainerHelpers.java   |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c89e15c/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 5eca572..0550895 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -190,7 +190,7 @@ public class AMContainerHelpers {
     // Construct the actual Container
     ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext(
         commonContainerSpec.getUser(), lResources, myEnv, commands,
-        myServiceData, commonContainerSpec.getContainerTokens().duplicate(),
+        myServiceData, commonContainerSpec.getTokens().duplicate(),
         acls);
 
     return container;


[38/50] [abbrv] git commit: TEZ-171: MapUtils should use the temp-dir passed in rather than env:PWD

Posted by ss...@apache.org.
TEZ-171: MapUtils should use the temp-dir passed in rather than env:PWD


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/5d0b8adf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/5d0b8adf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/5d0b8adf

Branch: refs/heads/master
Commit: 5d0b8adfb9390b67c53e9bbad7a9f0961300d8db
Parents: 6d5625f
Author: Mike Liddell <ml...@apache.org>
Authored: Thu May 30 16:45:30 2013 -0700
Committer: Mike Liddell <ml...@apache.org>
Committed: Thu May 30 16:45:30 2013 -0700

----------------------------------------------------------------------
 .../apache/tez/mapreduce/processor/MapUtils.java   |    3 +--
 1 files changed, 1 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5d0b8adf/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
index 48a1113..138a8e0 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
@@ -66,8 +66,7 @@ public class MapUtils {
     localSysDirs[0] = localDir;
 
     conf.setStrings(TezJobConfig.LOCAL_DIRS, localSysDirs);
-    conf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
-        System.getenv(Environment.PWD.name()));
+    conf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, localDir);
 
     LOG.info(TezJobConfig.LOCAL_DIRS + " for child: "
         + conf.get(TezJobConfig.LOCAL_DIRS));


[50/50] [abbrv] git commit: Merge branch 'master' into TEZ-1

Posted by ss...@apache.org.
Merge branch 'master' into TEZ-1


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/809bbd96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/809bbd96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/809bbd96

Branch: refs/heads/master
Commit: 809bbd9643427f7558669f6443b7b0e879191b8b
Parents: a10e607 d6cd24b
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jun 3 22:26:46 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jun 3 22:26:46 2013 -0700

----------------------------------------------------------------------

----------------------------------------------------------------------



[22/50] [abbrv] git commit: TEZ-9. Support querying state of the DAG AM from a client via a client RPC (bikas)

Posted by ss...@apache.org.
TEZ-9. Support querying state of the DAG AM from a client via a client RPC (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/ef37823c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/ef37823c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/ef37823c

Branch: refs/heads/master
Commit: ef37823c90efedb5a0b14f94b4e9e785cb1c5071
Parents: 37d7a4c
Author: Bikas Saha <bi...@apache.org>
Authored: Tue May 21 18:28:41 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue May 21 18:28:41 2013 -0700

----------------------------------------------------------------------
 tez-dag-api/pom.xml                                |   15 ++
 .../main/java/org/apache/tez/client/TezClient.java |   95 +++++++++++++
 .../org/apache/tez/dag/api/TezConfiguration.java   |   11 ++
 .../java/org/apache/tez/dag/api/TezException.java  |    2 +-
 .../org/apache/tez/dag/api/TezRemoteException.java |   31 +++++
 .../org/apache/tez/dag/api/client/DAGClient.java   |   10 +-
 .../org/apache/tez/dag/api/client/Progress.java    |    9 ++
 .../tez/dag/api/client/rpc/DAGClientRPCImpl.java   |   18 ++-
 .../apache/tez/dag/api/client/DAGClientServer.java |  106 +++++++++++++++
 .../DAGClientAMProtocolBlockingPBServerImpl.java   |  103 ++++++++++++++
 .../java/org/apache/tez/dag/app/DAGAppMaster.java  |   49 ++++---
 .../apache/tez/dag/app/client/ClientService.java   |   29 ----
 .../tez/dag/app/client/impl/TezClientService.java  |   43 ------
 .../apache/tez/dag/app/client/package-info.java    |   20 ---
 .../tez/dag/app/rm/TaskSchedulerEventHandler.java  |   31 +++--
 .../tez/mapreduce/ClientServiceDelegate.java       |   70 ++++++++--
 .../org/apache/tez/mapreduce/DAGJobStatus.java     |   27 ++++-
 17 files changed, 517 insertions(+), 152 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag-api/pom.xml b/tez-dag-api/pom.xml
index 74b1284..3cb0952 100644
--- a/tez-dag-api/pom.xml
+++ b/tez-dag-api/pom.xml
@@ -34,6 +34,10 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-common</artifactId>
     </dependency>
   </dependencies>
@@ -46,6 +50,17 @@
         <configuration>
         </configuration>
       </plugin>
+    <plugin>
+    <groupId>org.apache.maven.plugins</groupId>
+     <artifactId>maven-jar-plugin</artifactId>
+      <configuration>
+       <archive>
+         <manifest>
+           <mainClass>org.apache.tez.client.TezClient</mainClass>
+         </manifest>
+       </archive>
+     </configuration>
+    </plugin>
       <plugin>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-maven-plugins</artifactId>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
new file mode 100644
index 0000000..f794014
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -0,0 +1,95 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.client;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.client.YarnClient;
+import org.apache.hadoop.yarn.client.YarnClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezRemoteException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.client.rpc.DAGClientRPCImpl;
+
+public class TezClient {
+  private final TezConfiguration conf;
+  
+  public TezClient(TezConfiguration conf) {
+    this.conf = conf;
+  }
+
+  public DAGClient getDAGClient(String appIdStr) throws IOException, TezException {
+    try {
+      System.out.println("Fetching app: " + appIdStr);
+      ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
+      YarnClient yarnClient = new YarnClientImpl();
+      yarnClient.init(conf);
+      yarnClient.start();
+      ApplicationReport appReport = yarnClient.getApplicationReport(appId);
+      String host = appReport.getHost();
+      int port = appReport.getRpcPort();
+      return getDAGClient(host, port);
+    } catch (YarnRemoteException e) {
+      throw new TezException(e);
+    }
+  }
+  
+  public DAGClient getDAGClient(String host, int port) throws IOException {
+    System.out.println("App host port: " + host + ":" + port);
+    InetSocketAddress addr = new InetSocketAddress(host, port);
+    DAGClient dagClient;
+    dagClient = new DAGClientRPCImpl(1, addr, conf);
+    return dagClient;    
+  }
+  
+  public static void main(String[] args) {
+    try {
+      TezClient tezClient = new TezClient(
+          new TezConfiguration(new YarnConfiguration()));
+      DAGClient dagClient = tezClient.getDAGClient(args[1]);
+      String dagId = dagClient.getAllDAGs().get(0);
+      DAGStatus dagStatus = dagClient.getDAGStatus(dagId);
+      System.out.println("DAG: " + dagId + 
+                         " State: " + dagStatus.getState() +
+                         " Progress: " + dagStatus.getDAGProgress());
+      for(String vertexName : dagStatus.getVertexProgress().keySet()) {
+        System.out.println("VertexStatus from DagStatus:" +
+                           " Vertex: " + vertexName +
+                           " Progress: " + dagStatus.getVertexProgress().get(vertexName));
+        VertexStatus vertexStatus = dagClient.getVertexStatus(dagId, vertexName);
+        System.out.println("VertexStatus:" + 
+                           " Vertex: " + vertexName + 
+                           " State: " + vertexStatus.getState() + 
+                           " Progress: " + vertexStatus.getProgress());
+      }
+    } catch (Exception e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 597fb2f..4a407b3 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -65,6 +65,17 @@ public class TezConfiguration extends Configuration {
   public static final String DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD = TEZ_PREFIX
       + "node-blacklisting.ignore-threshold-node-percent";
   public static final int DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT = 33;
+  
+  /** Number of threads to handle job client RPC requests.*/
+  public static final String DAG_CLIENT_AM_THREAD_COUNT =
+                                    TEZ_PREFIX + "client.am.thread-count";
+  public static final int DAG_CLIENT_AM__THREAD_COUNT_DEFAULT = 1;
+  /** 
+   * Range of ports that the AM can use when binding. Leave blank
+   * if you want all possible ports.
+   */
+  public static final String DAG_CLIENT_AM_PORT_RANGE = 
+                                    TEZ_PREFIX + "client.am.port-range";
 
 
   public static final String DAG_AM_RESOURCE_MEMORY_MB = DAG_AM_PREFIX

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java
index 265b89c..3014309 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.api;
 
-/*
+/**
  * Base Tez Exception
  */
 public class TezException extends RuntimeException {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezRemoteException.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezRemoteException.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezRemoteException.java
new file mode 100644
index 0000000..8311852
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezRemoteException.java
@@ -0,0 +1,31 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api;
+
+/**
+ *  Base TezRemoteException
+ */
+public class TezRemoteException extends Exception {
+  private static final long serialVersionUID = 6337442733802964447L;
+  public TezRemoteException(Throwable cause) { super(cause); }
+  public TezRemoteException(String message) { super(message); }
+  public TezRemoteException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
index 477b017..0e9e55b 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
@@ -18,9 +18,10 @@
 
 package org.apache.tez.dag.api.client;
 
+import java.io.IOException;
 import java.util.List;
 
-import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezRemoteException;
 
 /*
  * Inteface class for monitoring the <code>DAG</code> running in a Tez DAG
@@ -31,16 +32,17 @@ public interface DAGClient {
   /**
    * Return the identifiers for all DAG's
    */
-  List<String> getAllDAGs() throws TezException;
+  List<String> getAllDAGs() throws IOException, TezRemoteException;
   
   /**
    * Get the status of a DAG
    */
-  DAGStatus getDAGStatus(String dagId) throws TezException;
+  DAGStatus getDAGStatus(String dagId) throws IOException, TezRemoteException;
   
   /**
    * Get the status of a Vertex of a DAG 
    */
   VertexStatus getVertexStatus(String dagId, 
-                               String vertexName) throws TezException;
+                               String vertexName) 
+                                   throws IOException, TezRemoteException;
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
index 7158e30..af9f2a4 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
@@ -47,5 +47,14 @@ public class Progress {
   public int getKilledTaskCount() {
     return proxy.getKilledTaskCount();
   }
+  
+  @Override
+  public String toString() {
+    return new String("Total: " + getTotalTaskCount() +
+                       " Succeeded: " + getSucceededTaskCount() +
+                       " Running: " + getRunningTaskCount() + 
+                       " Failed: " + getFailedTaskCount() + 
+                       " Killed: " + getKilledTaskCount());
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
index 9238f63..42a815e 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
@@ -26,7 +26,7 @@ import java.util.List;
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezRemoteException;
 import org.apache.tez.dag.api.client.DAGClient;
 import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.api.client.VertexStatus;
@@ -55,18 +55,20 @@ public class DAGClientRPCImpl implements DAGClient, Closeable {
   }
   
   @Override
-  public List<String> getAllDAGs() throws TezException {
+  public List<String> getAllDAGs() throws IOException, TezRemoteException {
     GetAllDAGsRequestProto requestProto = 
         GetAllDAGsRequestProto.newBuilder().build();
     try {
       return proxy.getAllDAGs(null, requestProto).getDagIdList();
     } catch (ServiceException e) {
-      throw new TezException(e);
+      // TEZ-151 retrieve wrapped TezRemoteException
+      throw new TezRemoteException(e);
     }
   }
 
   @Override
-  public DAGStatus getDAGStatus(String dagId) throws TezException {
+  public DAGStatus getDAGStatus(String dagId) 
+                                    throws IOException, TezRemoteException {
     GetDAGStatusRequestProto requestProto = 
         GetDAGStatusRequestProto.newBuilder().setDagId(dagId).build();
     
@@ -74,13 +76,14 @@ public class DAGClientRPCImpl implements DAGClient, Closeable {
       return new DAGStatus(
                  proxy.getDAGStatus(null, requestProto).getDagStatus());
     } catch (ServiceException e) {
-      throw new TezException(e);
+      // TEZ-151 retrieve wrapped TezRemoteException
+      throw new TezRemoteException(e);
     }
   }
 
   @Override
   public VertexStatus getVertexStatus(String dagId, String vertexName)
-      throws TezException {
+                                    throws IOException, TezRemoteException {
     GetVertexStatusRequestProto requestProto = 
         GetVertexStatusRequestProto.newBuilder().
                         setDagId(dagId).setVertexName(vertexName).build();
@@ -89,7 +92,8 @@ public class DAGClientRPCImpl implements DAGClient, Closeable {
       return new VertexStatus(
                  proxy.getVertexStatus(null, requestProto).getVertexStatus());
     } catch (ServiceException e) {
-      throw new TezException(e);
+      // TEZ-151 retrieve wrapped TezRemoteException
+      throw new TezRemoteException(e);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java
new file mode 100644
index 0000000..534fcaa
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java
@@ -0,0 +1,106 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPBServerImpl;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.DAGClientAMProtocol;
+
+import com.google.protobuf.BlockingService;
+
+public class DAGClientServer extends AbstractService {
+  static final Log LOG = LogFactory.getLog(DAGClientServer.class);
+      
+  DAGClient realInstance;
+  Server server;
+  InetSocketAddress bindAddress;
+
+  public DAGClientServer(DAGClient realInstance) {
+    super("DAGClientRPCServer");
+    this.realInstance = realInstance;
+  }
+  
+  @Override
+  public void start() {
+    try {
+      assert getConfig() instanceof TezConfiguration;
+      TezConfiguration conf = (TezConfiguration) getConfig();
+      InetSocketAddress addr = new InetSocketAddress(0);
+      
+      DAGClientAMProtocolBlockingPBServerImpl service = 
+          new DAGClientAMProtocolBlockingPBServerImpl(realInstance);
+      
+      BlockingService blockingService = 
+                DAGClientAMProtocol.newReflectiveBlockingService(service);
+      
+      int numHandlers = conf.getInt(TezConfiguration.DAG_CLIENT_AM_THREAD_COUNT, 
+                          TezConfiguration.DAG_CLIENT_AM__THREAD_COUNT_DEFAULT);
+      
+      String portRange = conf.get(TezConfiguration.DAG_CLIENT_AM_PORT_RANGE);
+      
+      server = createServer(DAGClientAMProtocolBlockingPB.class, addr, conf, 
+                            numHandlers, blockingService, portRange);
+      server.start();
+      bindAddress = NetUtils.getConnectAddress(server);
+      LOG.info("Instantiated DAGClientRPCServer at " + bindAddress);
+      super.start();
+    } catch (Exception e) {
+      LOG.error("Failed to start DAGClientServer: ", e);
+      throw new TezException(e);
+    }
+  }
+  
+  @Override
+  public void stop() {
+    if(server != null) {
+      server.stop();
+    }
+    super.stop();
+  }
+  
+  public InetSocketAddress getBindAddress() {
+    return bindAddress;
+  }
+  
+  private Server createServer(Class<?> pbProtocol, InetSocketAddress addr, Configuration conf, 
+      int numHandlers, 
+      BlockingService blockingService, String portRangeConfig) throws IOException {
+    RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class);
+    RPC.Server server = new RPC.Builder(conf).setProtocol(pbProtocol)
+        .setInstance(blockingService).setBindAddress(addr.getHostName())
+        .setPort(addr.getPort()).setNumHandlers(numHandlers).setVerbose(false)
+        .setPortRangeConfig(portRangeConfig)
+        .build();
+    server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, pbProtocol, blockingService);
+    return server;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java
new file mode 100644
index 0000000..81d7868
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java
@@ -0,0 +1,103 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client.rpc;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.tez.dag.api.TezRemoteException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.DAGStatusBuilder;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.client.VertexStatusBuilder;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAllDAGsRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAllDAGsResponseProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetDAGStatusRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetDAGStatusResponseProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetVertexStatusRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetVertexStatusResponseProto;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+
+public class DAGClientAMProtocolBlockingPBServerImpl implements
+    DAGClientAMProtocolBlockingPB {
+  
+  DAGClient real;
+  
+  public DAGClientAMProtocolBlockingPBServerImpl(DAGClient real) {
+    this.real = real;
+  }
+
+  @Override
+  public GetAllDAGsResponseProto getAllDAGs(RpcController controller,
+      GetAllDAGsRequestProto request) throws ServiceException {
+    try{
+      List<String> dagIds = real.getAllDAGs();
+      return GetAllDAGsResponseProto.newBuilder().addAllDagId(dagIds).build();
+    } catch(TezRemoteException e) {
+      throw wrapException(e);
+    } catch(IOException e) {
+      throw wrapException(e);
+    }
+  }
+
+  @Override
+  public GetDAGStatusResponseProto getDAGStatus(RpcController controller,
+      GetDAGStatusRequestProto request) throws ServiceException {
+    try {
+      String dagId = request.getDagId();
+      DAGStatus status;
+      status = real.getDAGStatus(dagId);
+      assert status instanceof DAGStatusBuilder;
+      DAGStatusBuilder builder = (DAGStatusBuilder) status;
+      return GetDAGStatusResponseProto.newBuilder().
+                                setDagStatus(builder.getProto()).build();
+    } catch (TezRemoteException e) {
+      throw wrapException(e);
+    } catch(IOException e) {
+      throw wrapException(e);
+    }
+  }
+
+  @Override
+  public GetVertexStatusResponseProto getVertexStatus(RpcController controller,
+      GetVertexStatusRequestProto request) throws ServiceException {
+    try {
+      String dagId = request.getDagId();
+      String vertexName = request.getVertexName();
+      VertexStatus status = real.getVertexStatus(dagId, vertexName);
+      assert status instanceof VertexStatusBuilder;
+      VertexStatusBuilder builder = (VertexStatusBuilder) status;
+      return GetVertexStatusResponseProto.newBuilder().
+                            setVertexStatus(builder.getProto()).build();
+    } catch (TezRemoteException e) {
+      throw wrapException(e);
+    } catch(IOException e) {
+      throw wrapException(e);
+    }
+  }
+  
+  ServiceException wrapException(Exception e){
+    return new ServiceException(e);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 804c20c..821dd71 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -67,13 +67,12 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGClientServer;
 import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.api.client.VertexStatus;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.app.client.ClientService;
-import org.apache.tez.dag.app.client.impl.TezClientService;
+import org.apache.tez.dag.api.TezRemoteException;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.Task;
 import org.apache.tez.dag.app.dag.TaskAttempt;
@@ -157,7 +156,6 @@ public class DAGAppMaster extends CompositeService {
   private AppContext context;
   private TezConfiguration conf; 
   private Dispatcher dispatcher;
-  private ClientService clientService;
   // TODO Recovery
   //private Recovery recoveryServ;
   private ContainerLauncher containerLauncher;
@@ -178,7 +176,9 @@ public class DAGAppMaster extends CompositeService {
   private HistoryEventHandler historyEventHandler;
 
   private DAGAppMasterState state;
-  private DAGMonitorServer monitor;
+  
+  DAGClientServer clientRpcServer;
+  private DAGClientHandler clientHandler;
 
   private DAG dag;
   private Credentials fsTokens = new Credentials(); // Filled during init
@@ -230,13 +230,16 @@ public class DAGAppMaster extends CompositeService {
 
     dagId = new TezDAGID(appAttemptID.getApplicationId(), 1);
     
-    monitor = new DAGMonitorServer();
+    clientHandler = new DAGClientHandler();
 
     // TODO Committer.
     //    committer = createOutputCommitter(conf);
 
     dispatcher = createDispatcher();
     addIfService(dispatcher);
+    
+    clientRpcServer = new DAGClientServer(clientHandler);
+    addIfService(clientRpcServer);
 
     taskHeartbeatHandler = createTaskHeartbeatHandler(context, conf);
     addIfService(taskHeartbeatHandler);
@@ -262,11 +265,6 @@ public class DAGAppMaster extends CompositeService {
     taskCleaner = createTaskCleaner(context);
     addIfService(taskCleaner);
 
-    // TODO TEZ-9
-    //service to handle requests from JobClient
-    clientService = new TezClientService();
-    addIfService(clientService);
-
     this.dagEventDispatcher = new DagEventDispatcher();
     this.vertexEventDispatcher = new VertexEventDispatcher();
 
@@ -287,7 +285,7 @@ public class DAGAppMaster extends CompositeService {
     dispatcher.register(NMCommunicatorEventType.class, containerLauncher);
 
     taskSchedulerEventHandler = new TaskSchedulerEventHandler(context,
-        clientService);
+        clientRpcServer);
     addIfService(taskSchedulerEventHandler);
     dispatcher.register(AMSchedulerEventType.class,
         taskSchedulerEventHandler);
@@ -712,30 +710,37 @@ public class DAGAppMaster extends CompositeService {
     LOG.info("On DAG completion. Old state: " + oldState + " new state: " + state);
   }
   
-  class DAGMonitorServer implements DAGClient {
+  class DAGClientHandler implements DAGClient {
 
     @Override
-    public List<String> getAllDAGs() {
+    public List<String> getAllDAGs() throws TezRemoteException {
       return Collections.singletonList(dag.getID().toString());
     }
 
     @Override
-    public DAGStatus getDAGStatus(String dagIdStr) throws TezException {
+    public DAGStatus getDAGStatus(String dagIdStr) 
+                                      throws IOException, TezRemoteException {
       return getDAG(dagIdStr).getDAGStatus();
     }
 
     @Override
-    public VertexStatus getVertexStatus(String dagIdStr, String vertexName) {
-      return getDAG(dagIdStr).getVertexStatus(vertexName);
+    public VertexStatus getVertexStatus(String dagIdStr, String vertexName) 
+        throws IOException, TezRemoteException{
+      VertexStatus status = getDAG(dagIdStr).getVertexStatus(vertexName);
+      if(status == null) {
+        throw new TezRemoteException("Unknown vertexName: " + vertexName);
+      }
+      
+      return status;
     }
     
-    DAG getDAG(String dagIdStr) {
+    DAG getDAG(String dagIdStr) throws IOException, TezRemoteException {
       TezDAGID dagId = TezDAGID.fromString(dagIdStr);
       if(dagId == null) {
-        throw new TezException("Bad dagId: " + dagIdStr);
+        throw new TezRemoteException("Bad dagId: " + dagIdStr);
       }
       if(!dagId.equals(dag.getID())) {
-        throw new TezException("Unknown dagId: " + dagIdStr);
+        throw new TezRemoteException("Unknown dagId: " + dagIdStr);
       }
       return dag;
     }
@@ -1083,7 +1088,9 @@ public class DAGAppMaster extends CompositeService {
       // that it doesnt take too long in shutting down
 
       // Signal the task scheduler.
-      appMaster.taskSchedulerEventHandler.setSignalled(true);
+      if(appMaster.getServiceState() == STATE.STARTED) {
+        appMaster.taskSchedulerEventHandler.setSignalled(true);
+      }
       if (EnumSet.of(DAGAppMasterState.NEW, DAGAppMasterState.INITED,
           DAGAppMasterState.RUNNING).contains(appMaster.state)) {
         // DAG not in a final state. Must have receive a KILL signal

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/app/client/ClientService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/client/ClientService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/client/ClientService.java
deleted file mode 100644
index 64937a9..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/client/ClientService.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.client;
-
-import java.net.InetSocketAddress;
-
-// TODONOTES - RPC service for clients
-public interface ClientService {
-
-  InetSocketAddress getBindAddress();
-
-  int getHttpPort();
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/app/client/impl/TezClientService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/client/impl/TezClientService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/client/impl/TezClientService.java
deleted file mode 100644
index 8fd79f0..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/client/impl/TezClientService.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.client.impl;
-
-import java.net.InetSocketAddress;
-
-import org.apache.tez.dag.app.client.ClientService;
-
-public class TezClientService implements ClientService {
-
-  // TODO remove dummy client service
-  private final InetSocketAddress dummySocketAddress =
-      new InetSocketAddress(0);
-
-  @Override
-  public InetSocketAddress getBindAddress() {
-    // TODO Auto-generated method stub
-    return dummySocketAddress;
-  }
-
-  @Override
-  public int getHttpPort() {
-    // TODO Auto-generated method stub
-    return 0;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/app/client/package-info.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/client/package-info.java b/tez-dag/src/main/java/org/apache/tez/dag/app/client/package-info.java
deleted file mode 100644
index a4cd9cc..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/client/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-@InterfaceAudience.Private
-package org.apache.tez.dag.app.client;
-import org.apache.hadoop.classification.InterfaceAudience;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 85ba0b2..c61cbcc 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -38,11 +38,11 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.tez.dag.api.client.DAGClientServer;
 import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.DAGAppMaster;
 import org.apache.tez.dag.app.DAGAppMasterState;
-import org.apache.tez.dag.app.client.ClientService;
 import org.apache.tez.dag.app.dag.TaskAttempt;
 import org.apache.tez.dag.app.dag.event.DAGAppMasterEvent;
 import org.apache.tez.dag.app.dag.event.DAGAppMasterEventType;
@@ -69,31 +69,24 @@ public class TaskSchedulerEventHandler extends AbstractService
   protected final AppContext appContext;
   @SuppressWarnings("rawtypes")
   private final EventHandler eventHandler;
-  private final TaskScheduler taskScheduler;
-  // TODO change this to DAGAppMaster
+  private TaskScheduler taskScheduler;
   private DAGAppMaster dagAppMaster;
   private Map<ApplicationAccessType, String> appAcls = null;
   private Thread eventHandlingThread;
   private volatile boolean stopEventHandling;
   // Has a signal (SIGTERM etc) been issued?
   protected volatile boolean isSignalled = false;
+  final DAGClientServer clientService;
 
   BlockingQueue<AMSchedulerEvent> eventQueue
                               = new LinkedBlockingQueue<AMSchedulerEvent>();
 
   public TaskSchedulerEventHandler(AppContext appContext,
-      ClientService clientService) {
+      DAGClientServer clientService) {
     super(TaskSchedulerEventHandler.class.getName());
     this.appContext = appContext;
-    eventHandler = appContext.getEventHandler();
-    InetSocketAddress serviceAddr = clientService.getBindAddress();
-    taskScheduler = 
-        new TaskScheduler(appContext.getApplicationAttemptId(),
-                          this,
-                          serviceAddr.getHostName(),
-                          serviceAddr.getPort(),
-                          serviceAddr.getHostName() + 
-                            ":" + clientService.getHttpPort());
+    this.eventHandler = appContext.getEventHandler();
+    this.clientService = clientService;
   }
   
   public Map<ApplicationAccessType, String> getApplicationAcls() {
@@ -346,14 +339,22 @@ public class TaskSchedulerEventHandler extends AbstractService
   @Override
   public synchronized void init(Configuration conf) {
     super.init(conf);
-    taskScheduler.init(conf);
-    // todo set heartbeat value from conf here
   }
   
   @Override
   public synchronized void start() {
     // FIXME hack alert how is this supposed to support multiple DAGs?
     // Answer: this is shared across dags. need job==app-dag-master
+    // TODO set heartbeat value from conf here
+    InetSocketAddress serviceAddr = clientService.getBindAddress();
+    taskScheduler = 
+        new TaskScheduler(appContext.getApplicationAttemptId(),
+                          this,
+                          serviceAddr.getHostName(),
+                          serviceAddr.getPort(),
+                          "");
+    taskScheduler.init(getConfig());
+
     dagAppMaster = appContext.getAppMaster();
     taskScheduler.start();
     this.eventHandlingThread = new Thread() {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
index 9cdc845..44ccc07 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
@@ -18,34 +18,47 @@
 
 package org.apache.tez.mapreduce;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.v2.LogParams;
 import org.apache.hadoop.mapreduce.TaskCompletionEvent;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezRemoteException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
 
 import java.io.IOException;
 
 public class ClientServiceDelegate {
+  private static final Log LOG = LogFactory.getLog(ClientServiceDelegate.class);
 
-  private final Configuration conf;
+  private final TezConfiguration conf;
   private final ResourceMgrDelegate rm;
+  private DAGClient dagClient;
+  private String currentDAGId;
+  private TezClient tezClient;
+  private ApplicationReport appReport;
 
   // FIXME
   // how to handle completed jobs that the RM does not know about?
 
   public ClientServiceDelegate(Configuration conf, ResourceMgrDelegate rm,
       JobID jobId) {
-    this.conf = new Configuration(conf); // Cloning for modifying.
+    this.conf = new TezConfiguration(conf); // Cloning for modifying.
     // For faster redirects from AM to HS.
     this.conf.setInt(
         CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
         this.conf.getInt(MRJobConfig.MR_CLIENT_TO_AM_IPC_MAX_RETRIES,
             MRJobConfig.DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES));
     this.rm = rm;
+    tezClient = new TezClient(this.conf);
   }
 
   public org.apache.hadoop.mapreduce.Counters getJobCounters(JobID jobId)
@@ -74,17 +87,42 @@ public class ClientServiceDelegate {
   }
   
   public JobStatus getJobStatus(JobID oldJobID) throws IOException {
-    org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
-      TypeConverter.toYarn(oldJobID);
-    ApplicationReport appReport;
     try {
-      appReport = rm.getApplicationReport(jobId.getAppId());
-    } catch (YarnRemoteException e) {
-      throw new IOException(e);
+      if(dagClient == null) {
+        appReport = getAppReport(oldJobID);
+        if(appReport.getYarnApplicationState() != YarnApplicationState.RUNNING) {
+          // if job not running return status from appReport;
+          return getJobStatusFromRM(appReport);
+        } else {
+          // job is running. create dag am client
+          dagClient = tezClient.getDAGClient(appReport.getHost(), 
+                                             appReport.getRpcPort());
+          currentDAGId = dagClient.getAllDAGs().get(0);
+        }
+      }
+      // return status from client. use saved appReport for queue etc
+      DAGStatus dagStatus = dagClient.getDAGStatus(currentDAGId);
+      return new DAGJobStatus(appReport, dagStatus);
+    } catch (TezRemoteException e) {
+      // AM not responding
+      dagClient = null;
+      currentDAGId = null;
+      appReport = getAppReport(oldJobID);
+      if(appReport.getYarnApplicationState() != YarnApplicationState.RUNNING) {
+        LOG.info("App not running. Falling back to RM for report.");
+      } else {
+        LOG.warn("App running but failed to get report from AM.", e);
+      }
     }
+    
+    // final fallback
+    return getJobStatusFromRM(appReport);
+  }
+  
+  private JobStatus getJobStatusFromRM(ApplicationReport appReport) {
     JobStatus jobStatus =
-        new DAGJobStatus(appReport);
-    return jobStatus;
+        new DAGJobStatus(appReport, null);
+    return jobStatus;            
   }
 
   public org.apache.hadoop.mapreduce.TaskReport[] getTaskReports(
@@ -114,4 +152,14 @@ public class ClientServiceDelegate {
     // FIXME logs for an attempt?
     throw new UnsupportedOperationException();
   }
+  
+  private ApplicationReport getAppReport(JobID oldJobID) throws IOException {
+    org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
+      TypeConverter.toYarn(oldJobID);
+    try {
+      return rm.getApplicationReport(jobId.getAppId());
+    } catch (YarnRemoteException e) {
+      throw new IOException(e);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
index d58cea4..08c61f8 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
@@ -32,14 +32,20 @@ import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.Progress;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+import org.mortbay.log.Log;
 
 public class DAGJobStatus extends JobStatus {
 
   private final ApplicationReport report;
+  private final DAGStatus dagStatus;
   
-  public DAGJobStatus(ApplicationReport appReport) {
+  public DAGJobStatus(ApplicationReport appReport, DAGStatus dagStatus) {
     super();
     this.report = appReport;
+    this.dagStatus = dagStatus;
   }
   
   @Override
@@ -129,6 +135,9 @@ public class DAGJobStatus extends JobStatus {
 
   @Override
   public synchronized float getMapProgress() {
+    if(dagStatus != null) {
+      return getProgress(MultiStageMRConfigUtil.getInitialMapVertexName());
+    }
     if (report.getYarnApplicationState().equals(
         YarnApplicationState.FINISHED)
         && report.getFinalApplicationStatus().equals(
@@ -160,6 +169,9 @@ public class DAGJobStatus extends JobStatus {
 
   @Override
   public synchronized float getReduceProgress() {
+    if(dagStatus != null) {
+      return getProgress(MultiStageMRConfigUtil.getFinalReduceVertexName());
+    }
     if (report.getYarnApplicationState().equals(
         YarnApplicationState.FINISHED)
         && report.getFinalApplicationStatus().equals(
@@ -350,5 +362,18 @@ public class DAGJobStatus extends JobStatus {
     buffer.append("needed-mem" + getNeededMem());
     return buffer.toString();
   }
+  
+  private float getProgress(String vertexName) {
+    Progress progress = dagStatus.getVertexProgress().get(vertexName);
+    if(progress == null) {
+      // no such stage. return 0 like MR app currently does.
+      return 0;
+    }
+    float totalTasks = (float) progress.getTotalTaskCount();
+    if(totalTasks != 0) {
+      return progress.getSucceededTaskCount()/totalTasks;
+    }
+    return 1;
+  }
 
 }


[33/50] [abbrv] git commit: TEZ-118. Create a MiniTezCluster for allowing of unit tests.

Posted by ss...@apache.org.
TEZ-118. Create a MiniTezCluster for allowing of unit tests.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/2340884c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/2340884c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/2340884c

Branch: refs/heads/master
Commit: 2340884c3ce85364ee1892909a58b82f532560db
Parents: 82ee45e
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed May 29 13:32:47 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Wed May 29 13:32:47 2013 -0700

----------------------------------------------------------------------
 pom.xml                                            |   26 +
 .../main/java/org/apache/tez/dag/api/Vertex.java   |   41 +-
 tez-dag/pom.xml                                    |   22 -
 tez-dag/src/test/resources/krb5.conf               |   28 -
 .../tez/mapreduce/examples/ExampleDriver.java      |   16 +-
 .../apache/tez/mapreduce/examples/MRRSleepJob.java |  460 +++++++++++++++
 tez-mapreduce-tests/pom.xml                        |  100 ++++
 .../apache/tez/mapreduce/MiniMRRTezCluster.java    |  151 +++++
 .../java/org/apache/tez/mapreduce/TestMRRJobs.java |  344 +++++++++++
 tez-mapreduce/pom.xml                              |   12 +
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |  108 ++--
 11 files changed, 1172 insertions(+), 136 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7fa336a..0e4f8e7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,6 +106,16 @@
       </dependency>
       <dependency>
         <groupId>org.apache.tez</groupId>
+        <artifactId>tez-mapreduce-examples</artifactId>
+        <version>${project.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.tez</groupId>
+        <artifactId>tez-mapreduce-tests</artifactId>
+        <version>${project.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.tez</groupId>
         <artifactId>tez-dag</artifactId>
         <version>${project.version}</version>
       </dependency>
@@ -191,6 +201,19 @@
         <version>${hadoop.version}</version>
       </dependency>
       <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-hdfs</artifactId>
+        <scope>test</scope>
+        <version>${hadoop.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-hdfs</artifactId>
+        <scope>test</scope>
+        <type>test-jar</type>
+        <version>${hadoop.version}</version>
+      </dependency>
+      <dependency>
         <groupId>org.mockito</groupId>
         <artifactId>mockito-all</artifactId>
         <version>1.9.5</version>
@@ -264,6 +287,7 @@
     <module>tez-yarn-client</module>
     <module>tez-mapreduce</module>
     <module>tez-mapreduce-examples</module>
+    <module>tez-mapreduce-tests</module>
     <module>tez-dag</module>
     <module>tez-dist</module>
   </modules>
@@ -299,7 +323,9 @@
             <forkMode>always</forkMode>
             <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
             <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
+            <redirectTestOutputToFile>true</redirectTestOutputToFile>
             <environmentVariables>
+              <JAVA_HOME>${java.home}</JAVA_HOME>
               <MALLOC_ARENA_MAX>4</MALLOC_ARENA_MAX>
             </environmentVariables>
             <systemPropertyVariables>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java
index 6bea664..cbcac16 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java
@@ -30,30 +30,30 @@ public class Vertex { // FIXME rename to Task
 
   private final String vertexName;
   private final String processorName;
-  
+
   private final int parallelism;
   private VertexLocationHint taskLocationsHint;
   private Resource taskResource;
   private Map<String, LocalResource> taskLocalResources;
   private Map<String, String> taskEnvironment;
-  
+
   private final List<Vertex> inputVertices = new ArrayList<Vertex>();
   private final List<Vertex> outputVertices = new ArrayList<Vertex>();
   private final List<String> inputEdgeIds = new ArrayList<String>();
   private final List<String> outputEdgeIds = new ArrayList<String>();
   private String javaOpts = "";
-  
-  
+
+
   public Vertex(String vertexName, String processorName, int parallelism) {
     this.vertexName = vertexName;
     this.processorName = processorName;
     this.parallelism = parallelism;
   }
-  
+
   public String getVertexName() { // FIXME rename to getName()
     return vertexName;
   }
-  
+
   public String getProcessorName() {
     return processorName;
   }
@@ -61,16 +61,19 @@ public class Vertex { // FIXME rename to Task
   public int getParallelism() {
     return parallelism;
   }
-  
+
   public void setTaskResource(Resource resource) {
     this.taskResource = resource;
   }
-  
+
   public Resource getTaskResource() {
     return taskResource;
   }
-  
+
   public void setTaskLocationsHint(TaskLocationHint[] locations) {
+    if (locations == null) {
+      return;
+    }
     assert locations.length == parallelism;
     taskLocationsHint = new VertexLocationHint(parallelism, locations);
   }
@@ -79,31 +82,31 @@ public class Vertex { // FIXME rename to Task
   VertexLocationHint getTaskLocationsHint() {
     return taskLocationsHint;
   }
-  
+
   public void setTaskLocalResources(Map<String, LocalResource> localResources) {
     this.taskLocalResources = localResources;
   }
-  
+
   public Map<String, LocalResource> getTaskLocalResources() {
     return taskLocalResources;
   }
-  
+
   public void setTaskEnvironment(Map<String, String> environment) {
     this.taskEnvironment = environment;
   }
-  
+
   public Map<String, String> getTaskEnvironment() {
     return taskEnvironment;
   }
-  
+
   public void setJavaOpts(String javaOpts){
      this. javaOpts = javaOpts;
   }
-  
+
   public String getJavaOpts(){
 	  return javaOpts;
   }
-  
+
   @Override
   public String toString() {
     return "[" + vertexName + " : " + processorName + "]";
@@ -118,15 +121,15 @@ public class Vertex { // FIXME rename to Task
     outputVertices.add(outputVertex);
     outputEdgeIds.add(edgeId);
   }
-  
+
   List<Vertex> getInputVertices() {
     return inputVertices;
   }
-  
+
   List<Vertex> getOutputVertices() {
     return outputVertices;
   }
-  
+
   List<String> getInputEdgeIds() {
     return inputEdgeIds;
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-dag/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index fc03bea..34194db 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -22,8 +22,6 @@
     <version>0.2.0-SNAPSHOT</version>
   </parent>
   <properties>
-    <maven.test.redirectTestOutputToFile>true
-    </maven.test.redirectTestOutputToFile>
     <tez.component>tez-dag</tez.component>
   </properties>
   <artifactId>tez-dag</artifactId>
@@ -102,26 +100,6 @@
   </dependencies>
 
   <build>
-    <pluginManagement>
-      <plugins>
-        <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-surefire-plugin</artifactId>
-          <configuration>
-            <forkMode>always</forkMode>
-            <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
-            <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
-            <environmentVariables>
-              <MALLOC_ARENA_MAX>4</MALLOC_ARENA_MAX>
-            </environmentVariables>
-            <systemPropertyVariables>
-              <java.security.krb5.conf>${basedir}/src/test/resources/krb5.conf
-              </java.security.krb5.conf>
-            </systemPropertyVariables>
-          </configuration>
-        </plugin>
-      </plugins>
-    </pluginManagement>
     <plugins>
       <plugin>
         <groupId>org.apache.rat</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-dag/src/test/resources/krb5.conf
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/resources/krb5.conf b/tez-dag/src/test/resources/krb5.conf
deleted file mode 100644
index 121ac6d..0000000
--- a/tez-dag/src/test/resources/krb5.conf
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# 
-[libdefaults]
-	default_realm = APACHE.ORG
-	udp_preference_limit = 1
-	extra_addresses = 127.0.0.1
-[realms]
-	APACHE.ORG = {
-		admin_server = localhost:88
-		kdc = localhost:88
-	}
-[domain_realm]
-	localhost = APACHE.ORG

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
index 94ba79f..468e0ac 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
@@ -24,23 +24,23 @@ import org.apache.tez.mapreduce.examples.terasort.TeraSort;
 import org.apache.tez.mapreduce.examples.terasort.TeraValidate;
 
 /**
- * A description of an example program based on its class and a 
+ * A description of an example program based on its class and a
  * human-readable description.
  */
 public class ExampleDriver {
-  
+
   public static void main(String argv[]){
     int exitCode = -1;
     ProgramDriver pgd = new ProgramDriver();
     try {
-      pgd.addClass("wordcount", WordCount.class, 
+      pgd.addClass("wordcount", WordCount.class,
           "A map/reduce program that counts the words in the input files.");
-      pgd.addClass("wordcountmrrtest", WordCountMRRTest.class, 
+      pgd.addClass("wordcountmrrtest", WordCountMRRTest.class,
           "A map/reduce program that counts the words in the input files."
           + " Map splits on spaces. First reduce splits on \".\"");
-      pgd.addClass("randomwriter", RandomWriter.class, 
+      pgd.addClass("randomwriter", RandomWriter.class,
           "A map/reduce program that writes 10GB of random data per node.");
-      pgd.addClass("randomtextwriter", RandomTextWriter.class, 
+      pgd.addClass("randomtextwriter", RandomTextWriter.class,
       "A map/reduce program that writes 10GB of random textual data per node.");
       pgd.addClass("sort", Sort.class,
           "A map/reduce program that sorts the data written by the random"
@@ -61,12 +61,14 @@ public class ExampleDriver {
           + " containing employee_name department name per line of input"
           + " and generates count of employees per department and"
           + " sorted on employee count");
+      pgd.addClass("mrrsleep", MRRSleepJob.class,
+          "MRR Sleep Job");
       exitCode = pgd.run(argv);
     }
     catch(Throwable e){
       e.printStackTrace();
     }
-    
+
     System.exit(exitCode);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
new file mode 100644
index 0000000..6115c6f
--- /dev/null
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -0,0 +1,460 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+
+/**
+ * Dummy class for testing MR framefork. Sleeps for a defined period
+ * of time in mapper and reducer. Generates fake input for map / reduce
+ * jobs. Note that generated number of input pairs is in the order
+ * of <code>numMappers * mapSleepTime / 100</code>, so the job uses
+ * some disk space.
+ */
+public class MRRSleepJob extends Configured implements Tool {
+
+  public static Log LOG = LogFactory.getLog(MRRSleepJob.class);
+
+  public static String MAP_SLEEP_COUNT = "mrr.sleepjob.map.sleep.count";
+  public static String REDUCE_SLEEP_COUNT =
+    "mrr.sleepjob.reduce.sleep.count";
+  public static String MAP_SLEEP_TIME = "mrr.sleepjob.map.sleep.time";
+  public static String REDUCE_SLEEP_TIME =
+    "mrr.sleepjob.reduce.sleep.time";
+  public static String IREDUCE_SLEEP_COUNT =
+      "mrr.sleepjob.ireduce.sleep.count";
+  public static String IREDUCE_SLEEP_TIME =
+      "mrr.sleepjob.ireduce.sleep.time";
+  public static String IREDUCE_STAGES_COUNT =
+      "mrr.sleepjob.ireduces.stages.count";
+
+  // Flags to inject failures
+  public static String MAP_THROW_ERROR = "mrr.sleepjob.map.throw.error";
+  public static String MAP_FATAL_ERROR = "mrr.sleepjob.map.fatal.error";
+  public static String MAP_ERROR_TASK_IDS = "mrr.sleepjob.map.error.task.ids";
+
+  public static class MRRSleepJobPartitioner extends
+      Partitioner<IntWritable, IntWritable> {
+    public int getPartition(IntWritable k, IntWritable v, int numPartitions) {
+      return k.get() % numPartitions;
+    }
+  }
+
+  public static class EmptySplit extends InputSplit implements Writable {
+    public void write(DataOutput out) throws IOException { }
+    public void readFields(DataInput in) throws IOException { }
+    public long getLength() { return 0L; }
+    public String[] getLocations() { return new String[0]; }
+  }
+
+  public static class SleepInputFormat
+      extends InputFormat<IntWritable,IntWritable> {
+
+    public List<InputSplit> getSplits(JobContext jobContext) {
+      List<InputSplit> ret = new ArrayList<InputSplit>();
+      int numSplits = jobContext.getConfiguration().
+                        getInt(MRJobConfig.NUM_MAPS, 1);
+      for (int i = 0; i < numSplits; ++i) {
+        ret.add(new EmptySplit());
+      }
+      return ret;
+    }
+
+    public RecordReader<IntWritable,IntWritable> createRecordReader(
+        InputSplit ignored, TaskAttemptContext taskContext)
+        throws IOException {
+      Configuration conf = taskContext.getConfiguration();
+      String vertexName = conf.get(
+          org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
+      boolean isIntermediateReduce =
+          MultiStageMRConfigUtil.getIntermediateStageNum(vertexName) != -1;
+
+      final int count =
+          (isIntermediateReduce)?
+              conf.getInt(IREDUCE_SLEEP_COUNT, 1) :
+              conf.getInt(MAP_SLEEP_COUNT, 1);
+      if (count < 0) throw new IOException("Invalid map count: " + count);
+
+      int totalIReduces = MultiStageMRConfigUtil.getNumIntermediateStages(conf);
+      boolean finalIReduce = totalIReduces ==
+          (MultiStageMRConfigUtil.getIntermediateStageNum(vertexName) + 1);
+
+      final int redcount = finalIReduce?
+          conf.getInt(REDUCE_SLEEP_COUNT, 1) :
+            conf.getInt(IREDUCE_SLEEP_COUNT, 1);
+      if (redcount < 0)
+        throw new IOException("Invalid reduce count: " + redcount);
+
+      final int emitPerMapTask = (redcount * taskContext.getNumReduceTasks());
+
+      return new RecordReader<IntWritable,IntWritable>() {
+        private int records = 0;
+        private int emitCount = 0;
+        private IntWritable key = null;
+        private IntWritable value = null;
+
+        public void initialize(InputSplit split, TaskAttemptContext context) {
+        }
+
+        public boolean nextKeyValue()
+            throws IOException {
+          if (count == 0) {
+            return false;
+          }
+          key = new IntWritable();
+          key.set(emitCount);
+          int emit = emitPerMapTask / count;
+          if ((emitPerMapTask) % count > records) {
+            ++emit;
+          }
+          emitCount += emit;
+          value = new IntWritable();
+          value.set(emit);
+          return records++ < count;
+        }
+        public IntWritable getCurrentKey() { return key; }
+        public IntWritable getCurrentValue() { return value; }
+        public void close() throws IOException { }
+        public float getProgress() throws IOException {
+          return count == 0 ? 100 : records / ((float)count);
+        }
+      };
+    }
+  }
+
+  public static class SleepMapper
+      extends Mapper<IntWritable, IntWritable, IntWritable, IntWritable> {
+    private long mapSleepDuration = 100;
+    private int mapSleepCount = 1;
+    private int count = 0;
+    private String vertexName;
+    private boolean throwError = false;
+    private boolean throwFatal = false;
+    private boolean finalAttempt = false;
+
+    protected void setup(Context context)
+      throws IOException, InterruptedException {
+      Configuration conf = context.getConfiguration();
+      this.mapSleepCount =
+        conf.getInt(MAP_SLEEP_COUNT, mapSleepCount);
+      this.mapSleepDuration = mapSleepCount == 0 ? 0 :
+        conf.getLong(MAP_SLEEP_TIME , 100) / mapSleepCount;
+      vertexName = conf.get(
+          org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
+
+      TaskAttemptID taId = context.getTaskAttemptID();
+      String[] taskIds = conf.getStrings(MAP_ERROR_TASK_IDS);
+      if (taId.getId()+1 >= context.getMaxMapAttempts()) {
+        finalAttempt = true;
+      }
+      boolean found = false;
+      if (taskIds != null) {
+        if (taskIds.length == 1 && taskIds[0].equals("*")) {
+          found = true;
+        }
+        if (!found) {
+          for (String taskId : taskIds) {
+            if (Integer.valueOf(taskId).intValue() ==
+                taId.getTaskID().getId()) {
+              found = true;
+              break;
+            }
+          }
+        }
+      }
+      if (found) {
+        if (!finalAttempt) {
+          throwError = conf.getBoolean(MAP_THROW_ERROR, false);
+        }
+        throwFatal = conf.getBoolean(MAP_FATAL_ERROR, false);
+      }
+    }
+
+    public void map(IntWritable key, IntWritable value, Context context
+               ) throws IOException, InterruptedException {
+      //it is expected that every map processes mapSleepCount number of records.
+      try {
+        LOG.info("Sleeping in InitialMap"
+            + ", vertexName=" + vertexName
+            + ", taskAttemptId=" + context.getTaskAttemptID()
+            + ", mapSleepDuration=" + mapSleepDuration
+            + ", mapSleepCount=" + mapSleepCount
+            + ", sleepLeft="
+            + (mapSleepDuration * (mapSleepCount - count)));
+        context.setStatus("Sleeping... (" +
+          (mapSleepDuration * (mapSleepCount - count)) + ") ms left");
+        Thread.sleep(mapSleepDuration);
+        if (throwError || throwFatal) {
+          throw new IOException("Throwing a simulated error from map");
+        }
+      }
+      catch (InterruptedException ex) {
+        throw (IOException)new IOException(
+            "Interrupted while sleeping").initCause(ex);
+      }
+      ++count;
+      // output reduceSleepCount * numReduce number of random values, so that
+      // each reducer will get reduceSleepCount number of keys.
+      int k = key.get();
+      for (int i = 0; i < value.get(); ++i) {
+        context.write(new IntWritable(k + i), new IntWritable(1));
+      }
+    }
+  }
+
+  public static class ISleepReducer
+  extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
+    private long iReduceSleepDuration = 100;
+    private int iReduceSleepCount = 1;
+    private int count = 0;
+    private String vertexName;
+
+    protected void setup(Context context)
+        throws IOException, InterruptedException {
+      Configuration conf = context.getConfiguration();
+      this.iReduceSleepCount =
+          conf.getInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
+      this.iReduceSleepDuration = iReduceSleepCount == 0 ? 0 :
+        conf.getLong(IREDUCE_SLEEP_TIME , 100) / iReduceSleepCount;
+      vertexName = conf.get(
+          org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
+    }
+
+    public void reduce(IntWritable key, Iterable<IntWritable> values,
+        Context context)
+            throws IOException, InterruptedException {
+      try {
+        LOG.info("Sleeping in IntermediateReduce"
+            + ", vertexName=" + vertexName
+            + ", taskAttemptId=" + context.getTaskAttemptID()
+            + ", iReduceSleepDuration=" + iReduceSleepDuration
+            + ", iReduceSleepCount=" + iReduceSleepCount
+            + ", sleepLeft="
+            + (iReduceSleepDuration * (iReduceSleepCount - count)));
+        context.setStatus("Sleeping... (" +
+          (iReduceSleepDuration * (iReduceSleepCount - count)) + ") ms left");
+        Thread.sleep(iReduceSleepDuration);
+      }
+      catch (InterruptedException ex) {
+        throw (IOException)new IOException(
+            "Interrupted while sleeping").initCause(ex);
+      }
+      ++count;
+      // output reduceSleepCount * numReduce number of random values, so that
+      // each reducer will get reduceSleepCount number of keys.
+      int k = key.get();
+      for (IntWritable value : values) {
+        for (int i = 0; i < value.get(); ++i) {
+          context.write(new IntWritable(k + i), new IntWritable(1));
+        }
+      }
+    }
+  }
+
+  public static class SleepReducer
+      extends Reducer<IntWritable, IntWritable, NullWritable, NullWritable> {
+    private long reduceSleepDuration = 100;
+    private int reduceSleepCount = 1;
+    private int count = 0;
+    private String vertexName;
+
+    protected void setup(Context context)
+      throws IOException, InterruptedException {
+      Configuration conf = context.getConfiguration();
+      this.reduceSleepCount =
+        conf.getInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
+      this.reduceSleepDuration = reduceSleepCount == 0 ? 0 :
+        conf.getLong(REDUCE_SLEEP_TIME , 100) / reduceSleepCount;
+      vertexName = conf.get(
+          org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
+    }
+
+    public void reduce(IntWritable key, Iterable<IntWritable> values,
+                       Context context)
+      throws IOException {
+      try {
+        LOG.info("Sleeping in FinalReduce"
+            + ", vertexName=" + vertexName
+            + ", taskAttemptId=" + context.getTaskAttemptID()
+            + ", reduceSleepDuration=" + reduceSleepDuration
+            + ", reduceSleepCount=" + reduceSleepCount
+            + ", sleepLeft="
+            + (reduceSleepDuration * (reduceSleepCount - count)));
+        context.setStatus("Sleeping... (" +
+            (reduceSleepDuration * (reduceSleepCount - count)) + ") ms left");
+        Thread.sleep(reduceSleepDuration);
+
+      }
+      catch (InterruptedException ex) {
+        throw (IOException)new IOException(
+          "Interrupted while sleeping").initCause(ex);
+      }
+      count++;
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    int res = ToolRunner.run(new Configuration(), new MRRSleepJob(), args);
+    System.exit(res);
+  }
+
+  public Job createJob(int numMapper, int numReducer, int iReduceStagesCount,
+      int numIReducer, long mapSleepTime, int mapSleepCount,
+      long reduceSleepTime, int reduceSleepCount,
+      long iReduceSleepTime, int iReduceSleepCount)
+      throws IOException {
+    Configuration conf = getConf();
+    conf.setLong(MAP_SLEEP_TIME, mapSleepTime);
+    conf.setLong(REDUCE_SLEEP_TIME, reduceSleepTime);
+    conf.setLong(IREDUCE_SLEEP_TIME, iReduceSleepTime);
+    conf.setInt(MAP_SLEEP_COUNT, mapSleepCount);
+    conf.setInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
+    conf.setInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
+    conf.setInt(MRJobConfig.NUM_MAPS, numMapper);
+    conf.setInt(IREDUCE_STAGES_COUNT, iReduceStagesCount);
+
+    // Configure intermediate reduces
+    conf.setInt(
+        org.apache.tez.mapreduce.hadoop.MRJobConfig.MRR_INTERMEDIATE_STAGES,
+        iReduceStagesCount);
+    LOG.info("Running MRR with " + iReduceStagesCount + " IR stages");
+
+    for (int i = 0; i < iReduceStagesCount; ++i) {
+      // Set reducer class for intermediate reduce
+      conf.setClass(
+          MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+          "mapreduce.job.reduce.class"), ISleepReducer.class, Reducer.class);
+      // Set reducer output key class
+      conf.setClass(
+          MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+          "mapreduce.map.output.key.class"), IntWritable.class, Object.class);
+      // Set reducer output value class
+      conf.setClass(
+          MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+          "mapreduce.map.output.value.class"), IntWritable.class, Object.class);
+      conf.setInt(
+          MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+          "mapreduce.job.reduces"), numIReducer);
+    }
+
+    Job job = Job.getInstance(conf, "sleep");
+    job.setNumReduceTasks(numReducer);
+    job.setJarByClass(MRRSleepJob.class);
+    job.setNumReduceTasks(numReducer);
+    job.setMapperClass(SleepMapper.class);
+    job.setMapOutputKeyClass(IntWritable.class);
+    job.setMapOutputValueClass(IntWritable.class);
+    job.setReducerClass(SleepReducer.class);
+    job.setOutputFormatClass(NullOutputFormat.class);
+    job.setInputFormatClass(SleepInputFormat.class);
+    job.setPartitionerClass(MRRSleepJobPartitioner.class);
+    job.setSpeculativeExecution(false);
+    job.setJobName("Sleep job");
+
+
+    FileInputFormat.addInputPath(job, new Path("ignored"));
+    return job;
+  }
+
+  public int run(String[] args) throws Exception {
+
+    if(args.length < 1) {
+      System.err.println("MRRSleepJob [-m numMapper] [-r numReducer]" +
+          " [-ir numIntermediateReducer]" +
+          " [-irs numIntermediateReducerStages]" +
+          " [-mt mapSleepTime (msec)] [-rt reduceSleepTime (msec)]" +
+          " [-irt intermediateReduceSleepTime]" +
+          " [-recordt recordSleepTime (msec)]");
+      ToolRunner.printGenericCommandUsage(System.err);
+      return 2;
+    }
+
+    int numMapper = 1, numReducer = 1, numIReducer = 1;
+    long mapSleepTime = 100, reduceSleepTime = 100, recSleepTime = 100,
+        iReduceSleepTime=1;
+    int mapSleepCount = 1, reduceSleepCount = 1, iReduceSleepCount = 1;
+    int iReduceStagesCount = 1;
+
+    for(int i=0; i < args.length; i++ ) {
+      if(args[i].equals("-m")) {
+        numMapper = Integer.parseInt(args[++i]);
+      }
+      else if(args[i].equals("-r")) {
+        numReducer = Integer.parseInt(args[++i]);
+      }
+      else if(args[i].equals("-ir")) {
+        numIReducer = Integer.parseInt(args[++i]);
+      }
+      else if(args[i].equals("-mt")) {
+        mapSleepTime = Long.parseLong(args[++i]);
+      }
+      else if(args[i].equals("-rt")) {
+        reduceSleepTime = Long.parseLong(args[++i]);
+      }
+      else if(args[i].equals("-irt")) {
+        iReduceSleepTime = Long.parseLong(args[++i]);
+      }
+      else if(args[i].equals("-irs")) {
+        iReduceStagesCount = Integer.parseInt(args[++i]);
+      }
+      else if (args[i].equals("-recordt")) {
+        recSleepTime = Long.parseLong(args[++i]);
+      }
+    }
+
+    // sleep for *SleepTime duration in Task by recSleepTime per record
+    mapSleepCount = (int)Math.ceil(mapSleepTime / ((double)recSleepTime));
+    reduceSleepCount = (int)Math.ceil(reduceSleepTime / ((double)recSleepTime));
+    iReduceSleepCount = (int)Math.ceil(iReduceSleepTime / ((double)recSleepTime));
+    Job job = createJob(numMapper, numReducer, iReduceStagesCount, numIReducer,
+        mapSleepTime, mapSleepCount, reduceSleepTime, reduceSleepCount,
+        iReduceSleepTime, iReduceSleepCount);
+    return job.waitForCompletion(true) ? 0 : 1;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/pom.xml b/tez-mapreduce-tests/pom.xml
new file mode 100644
index 0000000..0fb1024
--- /dev/null
+++ b/tez-mapreduce-tests/pom.xml
@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.tez</groupId>
+    <artifactId>tez</artifactId>
+    <version>0.2.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>tez-mapreduce-tests</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-dag</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-mapreduce</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-yarn-client</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-mapreduce-examples</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-tests</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
new file mode 100644
index 0000000..41a1cf3
--- /dev/null
+++ b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
@@ -0,0 +1,151 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.mapreduce;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.ShuffleHandler;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
+import org.apache.hadoop.util.JarFinder;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.hadoop.yarn.service.Service;
+import org.apache.tez.dag.app.DAGAppMaster;
+import org.apache.tez.mapreduce.hadoop.MRConfig;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+
+/**
+ * Configures and starts the Tez-specific components in the YARN cluster.
+ *
+ * When using this mini cluster, the user is expected to
+ */
+public class MiniMRRTezCluster extends MiniYARNCluster {
+
+  public static final String APPJAR = JarFinder.getJar(DAGAppMaster.class);
+
+  private static final Log LOG = LogFactory.getLog(MiniMRRTezCluster.class);
+
+  private static final String YARN_CLUSTER_CONFIG = "yarn-site.xml";
+
+  private Path confFilePath;
+
+  public MiniMRRTezCluster(String testName) {
+    this(testName, 1);
+  }
+
+  public MiniMRRTezCluster(String testName, int noOfNMs) {
+    super(testName, noOfNMs, 4, 4);
+  }
+
+  public MiniMRRTezCluster(String testName, int noOfNMs,
+      int numLocalDirs, int numLogDirs)  {
+    super(testName, noOfNMs, numLocalDirs, numLogDirs);
+  }
+
+  @Override
+  public void init(Configuration conf) {
+    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_TEZ_FRAMEWORK_NAME);
+    if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
+      conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(),
+          "apps_staging_dir" + Path.SEPARATOR).getAbsolutePath());
+    }
+
+    // VMEM monitoring disabled, PMEM monitoring enabled.
+    conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
+    conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
+
+    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,  "000");
+
+    try {
+      Path stagingPath = FileContext.getFileContext(conf).makeQualified(
+          new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
+      FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf);
+      if (fc.util().exists(stagingPath)) {
+        LOG.info(stagingPath + " exists! deleting...");
+        fc.delete(stagingPath, true);
+      }
+      LOG.info("mkdir: " + stagingPath);
+      fc.mkdir(stagingPath, null, true);
+
+      //mkdir done directory as well
+      String doneDir =
+          JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
+      Path doneDirPath = fc.makeQualified(new Path(doneDir));
+      fc.mkdir(doneDirPath, null, true);
+    } catch (IOException e) {
+      throw new YarnException("Could not create staging directory. ", e);
+    }
+    conf.set(MRConfig.MASTER_ADDRESS, "test");
+
+    //configure the shuffle service in NM
+    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
+        new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
+    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
+        ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
+        Service.class);
+
+    // Non-standard shuffle port
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+
+    conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
+        DefaultContainerExecutor.class, ContainerExecutor.class);
+
+    // TestMRJobs is for testing non-uberized operation only; see TestUberAM
+    // for corresponding uberized tests.
+    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
+
+    super.init(conf);
+  }
+
+  @Override
+  public void start() {
+    super.start();
+    File workDir = super.getTestWorkDir();
+    Configuration conf = super.getConfig();
+
+    confFilePath = new Path(workDir.getAbsolutePath(), YARN_CLUSTER_CONFIG);
+    File confFile = new File(confFilePath.toString());
+    try {
+      confFile.createNewFile();
+      conf.writeXml(new FileOutputStream(confFile));
+      confFile.deleteOnExit();
+    } catch (IOException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+      throw new RuntimeException(e);
+    }
+    confFilePath = new Path(confFile.getAbsolutePath());
+  }
+
+  public Path getConfigFilePath() {
+    return confFilePath;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java
new file mode 100644
index 0000000..520a1ac
--- /dev/null
+++ b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java
@@ -0,0 +1,344 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.mapreduce;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.RandomTextWriterJob;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobCounter;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskReport;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.JarFinder;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.tez.mapreduce.examples.MRRSleepJob;
+import org.apache.tez.mapreduce.hadoop.MRConfig;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestMRRJobs {
+
+  private static final Log LOG = LogFactory.getLog(TestMRRJobs.class);
+
+  protected static MiniMRRTezCluster mrrTezCluster;
+  protected static MiniDFSCluster dfsCluster;
+
+  private static Configuration conf = new Configuration();
+  private static FileSystem localFs;
+  private static FileSystem remoteFs;
+  static {
+    try {
+      localFs = FileSystem.getLocal(conf);
+    } catch (IOException io) {
+      throw new RuntimeException("problem getting local fs", io);
+    }
+  }
+
+  private static String TEST_ROOT_DIR = "target"
+      + Path.SEPARATOR + TestMRRJobs.class.getName() + "-tmpDir";
+
+  private static Path TEST_ROOT_DIR_PATH =
+      localFs.makeQualified(new Path(TEST_ROOT_DIR));
+  static Path APP_JAR = new Path(TEST_ROOT_DIR_PATH, "MRAppJar.jar");
+  static Path YARN_SITE_XML = new Path(TEST_ROOT_DIR_PATH, "yarn-site.xml");
+  private static final String OUTPUT_ROOT_DIR = "/tmp" + Path.SEPARATOR +
+      TestMRRJobs.class.getSimpleName();
+
+  @BeforeClass
+  public static void setup() throws IOException {
+    try {
+      conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
+      dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
+        .format(true).racks(null).build();
+      remoteFs = dfsCluster.getFileSystem();
+    } catch (IOException io) {
+      throw new RuntimeException("problem starting mini dfs cluster", io);
+    }
+
+    if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+      LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
+               + " not found. Not running test.");
+      return;
+    }
+
+    if (mrrTezCluster == null) {
+      mrrTezCluster = new MiniMRRTezCluster(TestMRRJobs.class.getName(), 1,
+          1, 1);
+      Configuration conf = new Configuration();
+      conf.set("fs.defaultFS", remoteFs.getUri().toString());   // use HDFS
+      conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir");
+      mrrTezCluster.init(conf);
+      mrrTezCluster.start();
+    }
+
+    localFs.copyFromLocalFile(new Path(MiniMRRTezCluster.APPJAR), APP_JAR);
+    localFs.setPermission(APP_JAR, new FsPermission("700"));
+    localFs.copyFromLocalFile(mrrTezCluster.getConfigFilePath(), YARN_SITE_XML);
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    if (mrrTezCluster != null) {
+      mrrTezCluster.stop();
+      mrrTezCluster = null;
+    }
+    if (dfsCluster != null) {
+      dfsCluster.shutdown();
+      dfsCluster = null;
+    }
+  }
+
+  @Test (timeout = 300000)
+  public void testMRRSleepJob() throws IOException, InterruptedException,
+      ClassNotFoundException {
+    LOG.info("\n\n\nStarting testMRRSleepJob().");
+
+    if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+      LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
+               + " not found. Not running test.");
+      return;
+    }
+
+    Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());
+
+    MRRSleepJob sleepJob = new MRRSleepJob();
+    sleepJob.setConf(sleepConf);
+
+    Job job = sleepJob.createJob(1, 1, 1, 1, 1,
+        1, 1, 1, 1, 1);
+
+    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+    job.addFileToClassPath(YARN_SITE_XML);
+    job.setJarByClass(MRRSleepJob.class);
+    job.setMaxMapAttempts(1); // speed up failures
+    job.submit();
+    String trackingUrl = job.getTrackingURL();
+    String jobId = job.getJobID().toString();
+    boolean succeeded = job.waitForCompletion(true);
+    Assert.assertTrue(succeeded);
+    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
+    Assert.assertTrue("Tracking URL was " + trackingUrl +
+                      " but didn't Match Job ID " + jobId ,
+          trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));
+
+    // FIXME once counters and task progress can be obtained properly
+    // TODO use dag client to test counters and task progress?
+    // what about completed jobs?
+
+  }
+
+  @Test (timeout = 60000)
+  public void testRandomWriter() throws IOException, InterruptedException,
+      ClassNotFoundException {
+
+    LOG.info("\n\n\nStarting testRandomWriter().");
+    if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+      LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
+               + " not found. Not running test.");
+      return;
+    }
+
+    RandomTextWriterJob randomWriterJob = new RandomTextWriterJob();
+    mrrTezCluster.getConfig().set(RandomTextWriterJob.TOTAL_BYTES, "3072");
+    mrrTezCluster.getConfig().set(RandomTextWriterJob.BYTES_PER_MAP, "1024");
+    Job job = randomWriterJob.createJob(mrrTezCluster.getConfig());
+    Path outputDir = new Path(OUTPUT_ROOT_DIR, "random-output");
+    FileOutputFormat.setOutputPath(job, outputDir);
+    job.setSpeculativeExecution(false);
+    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+    job.addFileToClassPath(YARN_SITE_XML);
+    job.setJarByClass(RandomTextWriterJob.class);
+    job.setMaxMapAttempts(1); // speed up failures
+    job.submit();
+    String trackingUrl = job.getTrackingURL();
+    String jobId = job.getJobID().toString();
+    boolean succeeded = job.waitForCompletion(true);
+    Assert.assertTrue(succeeded);
+    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
+    Assert.assertTrue("Tracking URL was " + trackingUrl +
+                      " but didn't Match Job ID " + jobId ,
+          trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));
+
+    // Make sure there are three files in the output-dir
+
+    RemoteIterator<FileStatus> iterator =
+        FileContext.getFileContext(mrrTezCluster.getConfig()).listStatus(
+            outputDir);
+    int count = 0;
+    while (iterator.hasNext()) {
+      FileStatus file = iterator.next();
+      if (!file.getPath().getName()
+          .equals(FileOutputCommitter.SUCCEEDED_FILE_NAME)) {
+        count++;
+      }
+    }
+    Assert.assertEquals("Number of part files is wrong!", 3, count);
+
+  }
+
+
+  @Test (timeout = 60000)
+  public void testFailingJob() throws IOException, InterruptedException,
+      ClassNotFoundException {
+
+    LOG.info("\n\n\nStarting testFailingMapper().");
+
+    if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+      LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
+               + " not found. Not running test.");
+      return;
+    }
+
+    Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());
+
+    MRRSleepJob sleepJob = new MRRSleepJob();
+    sleepJob.setConf(sleepConf);
+
+    Job job = sleepJob.createJob(1, 1, 1, 1, 1,
+        1, 1, 1, 1, 1);
+
+    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+    job.addFileToClassPath(YARN_SITE_XML);
+    job.setJarByClass(MRRSleepJob.class);
+    job.setMaxMapAttempts(1); // speed up failures
+    job.getConfiguration().setBoolean(MRRSleepJob.MAP_FATAL_ERROR, true);
+    job.getConfiguration().set(MRRSleepJob.MAP_ERROR_TASK_IDS, "*");
+
+    job.submit();
+    boolean succeeded = job.waitForCompletion(true);
+    Assert.assertFalse(succeeded);
+    Assert.assertEquals(JobStatus.State.FAILED, job.getJobState());
+
+    // FIXME once counters and task progress can be obtained properly
+    // TODO verify failed task diagnostics
+  }
+
+  @Test (timeout = 300000)
+  public void testFailingAttempt() throws IOException, InterruptedException,
+      ClassNotFoundException {
+
+    LOG.info("\n\n\nStarting testFailingMapper().");
+
+    if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+      LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
+               + " not found. Not running test.");
+      return;
+    }
+
+    Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());
+
+    MRRSleepJob sleepJob = new MRRSleepJob();
+    sleepJob.setConf(sleepConf);
+
+    Job job = sleepJob.createJob(1, 1, 1, 1, 1,
+        1, 1, 1, 1, 1);
+
+    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+    job.addFileToClassPath(YARN_SITE_XML);
+    job.setJarByClass(MRRSleepJob.class);
+    job.setMaxMapAttempts(3); // speed up failures
+    job.getConfiguration().setBoolean(MRRSleepJob.MAP_THROW_ERROR, true);
+    job.getConfiguration().set(MRRSleepJob.MAP_ERROR_TASK_IDS, "0");
+
+    job.submit();
+    boolean succeeded = job.waitForCompletion(true);
+    Assert.assertTrue(succeeded);
+    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
+
+    // FIXME once counters and task progress can be obtained properly
+    // TODO verify failed task diagnostics
+  }
+
+
+  /*
+  //@Test (timeout = 60000)
+  public void testMRRSleepJobWithSecurityOn() throws IOException,
+      InterruptedException, ClassNotFoundException {
+
+    LOG.info("\n\n\nStarting testMRRSleepJobWithSecurityOn().");
+
+    if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+      return;
+    }
+
+    mrrTezCluster.getConfig().set(
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+        "kerberos");
+    mrrTezCluster.getConfig().set(YarnConfiguration.RM_KEYTAB, "/etc/krb5.keytab");
+    mrrTezCluster.getConfig().set(YarnConfiguration.NM_KEYTAB, "/etc/krb5.keytab");
+    mrrTezCluster.getConfig().set(YarnConfiguration.RM_PRINCIPAL,
+        "rm/sightbusy-lx@LOCALHOST");
+    mrrTezCluster.getConfig().set(YarnConfiguration.NM_PRINCIPAL,
+        "nm/sightbusy-lx@LOCALHOST");
+
+    UserGroupInformation.setConfiguration(mrrTezCluster.getConfig());
+
+    // Keep it in here instead of after RM/NM as multiple user logins happen in
+    // the same JVM.
+    UserGroupInformation user = UserGroupInformation.getCurrentUser();
+
+    LOG.info("User name is " + user.getUserName());
+    for (Token<? extends TokenIdentifier> str : user.getTokens()) {
+      LOG.info("Token is " + str.encodeToUrlString());
+    }
+    user.doAs(new PrivilegedExceptionAction<Void>() {
+      @Override
+      public Void run() throws Exception {
+        MRRSleepJob sleepJob = new MRRSleepJob();
+        sleepJob.setConf(mrrTezCluster.getConfig());
+        Job job = sleepJob.createJob(3, 0, 10000, 1, 0, 0);
+        // //Job with reduces
+        // Job job = sleepJob.createJob(3, 2, 10000, 1, 10000, 1);
+        job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+        job.submit();
+        String trackingUrl = job.getTrackingURL();
+        String jobId = job.getJobID().toString();
+        job.waitForCompletion(true);
+        Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
+        Assert.assertTrue("Tracking URL was " + trackingUrl +
+                          " but didn't Match Job ID " + jobId ,
+          trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));
+        return null;
+      }
+    });
+
+    // TODO later:  add explicit "isUber()" checks of some sort
+  }
+  */
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/tez-mapreduce/pom.xml b/tez-mapreduce/pom.xml
index e785f6b..e32b098 100644
--- a/tez-mapreduce/pom.xml
+++ b/tez-mapreduce/pom.xml
@@ -54,6 +54,18 @@
       <artifactId>hadoop-mapreduce-client-common</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-tests</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-all</artifactId>
       <scope>test</scope>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index a79a4f9..801ce8a 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -126,6 +126,7 @@ public class YARNRunner implements ClientProtocol {
 
   final public static FsPermission DAG_FILE_PERMISSION =
       FsPermission.createImmutable((short) 0644);
+  final public static int UTF8_CHUNK_SIZE = 16 * 1024;
 
   /**
    * Yarn runner incapsulates the client interface of
@@ -395,7 +396,7 @@ public class YARNRunner implements ClientProtocol {
     }
   }
 
-  
+
   private void setupMapReduceEnv(Configuration jobConf,
       Map<String, String> environment, boolean isMap) throws IOException {
 
@@ -470,7 +471,7 @@ public class YARNRunner implements ClientProtocol {
           + "=" + entry.getValue());
     }
   }
-  
+
   private Vertex configureIntermediateReduceStage(FileSystem fs, JobID jobId,
       Configuration jobConf, String jobSubmitDir, Credentials ts,
       Map<String, LocalResource> jobLocalResources, int iReduceIndex)
@@ -482,7 +483,7 @@ public class YARNRunner implements ClientProtocol {
     Vertex vertex = new Vertex(
         MultiStageMRConfigUtil.getIntermediateStageVertexName(stageNum),
         "org.apache.tez.mapreduce.processor.reduce.ReduceProcessor", numTasks);
-    
+
     Map<String, String> reduceEnv = new HashMap<String, String>();
     setupMapReduceEnv(conf, reduceEnv, false);
 
@@ -501,7 +502,7 @@ public class YARNRunner implements ClientProtocol {
     vertex.setTaskResource(reduceResource);
 
     vertex.setJavaOpts(getReduceJavaOpts(conf));
-    
+
     return vertex;
   }
 
@@ -518,7 +519,7 @@ public class YARNRunner implements ClientProtocol {
       vertices[i] = configureIntermediateReduceStage(fs, jobId, jobConf, jobSubmitDir, ts,
           jobLocalResources, i);
       dag.addVertex(vertices[i]);
-      
+
       LOG.info("XXXX Adding intermediate vertex to DAG"
           + ", vertexName=" + vertices[i].getVertexName()
           + ", processor=" + vertices[i].getProcessorName()
@@ -550,7 +551,7 @@ public class YARNRunner implements ClientProtocol {
     String mapProcessor = "org.apache.tez.mapreduce.processor.map.MapProcessor";
     Vertex mapVertex = new Vertex(
         MultiStageMRConfigUtil.getInitialMapVertexName(),
-        mapProcessor, numMaps); 
+        mapProcessor, numMaps);
 
     // FIXME set up map environment
     Map<String, String> mapEnv = new HashMap<String, String>();
@@ -575,7 +576,7 @@ public class YARNRunner implements ClientProtocol {
     mapVertex.setTaskResource(mapResource);
 
     mapVertex.setJavaOpts(getMapJavaOpts(jobConf));
-    
+
     LOG.info("XXXX Adding map vertex to DAG"
         + ", vertexName=" + mapVertex.getVertexName()
         + ", processor=" + mapVertex.getProcessorName()
@@ -618,7 +619,7 @@ public class YARNRunner implements ClientProtocol {
       reduceVertex.setTaskResource(reduceResource);
 
       reduceVertex.setJavaOpts(getReduceJavaOpts(jobConf));
-      
+
       LOG.info("XXXX Adding reduce vertex to DAG"
           + ", vertexName=" + reduceVertex.getVertexName()
           + ", processor=" + reduceVertex.getProcessorName()
@@ -677,11 +678,11 @@ public class YARNRunner implements ClientProtocol {
         dag.addConfiguration(entry.getValue(), mrConf.get(entry.getKey()));
       }
     }
-    
+
     String jobName = mrConf.get(MRJobConfig.JOB_NAME);
     if(jobName != null) {
       dag.setName(jobName);
-    }    
+    }
   }
 
   private ApplicationSubmissionContext createApplicationSubmissionContext(
@@ -712,13 +713,13 @@ public class YARNRunner implements ClientProtocol {
     vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
 
 //[Debug AppMaster] Current simplest way to attach debugger to AppMaster
-// Uncomment the following, then launch a regular job, eg 
-// >hadoop jar {path}\hadoop-mapreduce-examples-3.0.0-SNAPSHOT.jar pi 2 2                  
+// Uncomment the following, then launch a regular job, eg
+// >hadoop jar {path}\hadoop-mapreduce-examples-3.0.0-SNAPSHOT.jar pi 2 2
 //     LOG.error(" !!!!!!!!!");
 //     LOG.error(" !!!!!!!!! Launching AppMaster in debug/suspend mode.  Attach to port 8002");
 //     LOG.error(" !!!!!!!!!");
 //     vargs.add("-Xdebug -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=8002,server=y,suspend=y");
-    
+
     // TODO -Dtez.root.logger??
     String amLogLevel = jobConf.get(MRJobConfig.MR_AM_LOG_LEVEL,
         MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
@@ -782,8 +783,10 @@ public class YARNRunner implements ClientProtocol {
     DAGPlan dagPB = dag.createDag();
     FSDataOutputStream dagPBOutBinaryStream = null;
     FSDataOutputStream dagPBOutTextStream = null;
-    Path binaryPath =  new Path(jobSubmitDir, TezConfiguration.DAG_AM_PLAN_PB_BINARY);
-    Path textPath =  new Path(jobSubmitDir, TezConfiguration.DAG_AM_PLAN_PB_TEXT);
+    Path binaryPath =  new Path(jobSubmitDir,
+        TezConfiguration.DAG_AM_PLAN_PB_BINARY);
+    Path textPath =  new Path(jobSubmitDir,
+        TezConfiguration.DAG_AM_PLAN_PB_TEXT);
     try {
       //binary output
       dagPBOutBinaryStream = FileSystem.create(fs, binaryPath,
@@ -793,7 +796,22 @@ public class YARNRunner implements ClientProtocol {
       // text / human-readable output
       dagPBOutTextStream = FileSystem.create(fs, textPath,
           new FsPermission(DAG_FILE_PERMISSION));
-      dagPBOutTextStream.writeUTF(dagPB.toString());
+      String dagPBStr = dagPB.toString();
+      int dagPBStrLen = dagPBStr.length();
+      if (dagPBStrLen <= UTF8_CHUNK_SIZE) {
+        dagPBOutTextStream.writeUTF(dagPBStr);
+      } else {
+        int startIndex = 0;
+        while (startIndex < dagPBStrLen) {
+          int endIndex = startIndex + UTF8_CHUNK_SIZE;
+          if (endIndex > dagPBStrLen) {
+            endIndex = dagPBStrLen;
+          }
+          dagPBOutTextStream.writeUTF(
+              dagPBStr.substring(startIndex, endIndex));
+          startIndex += UTF8_CHUNK_SIZE;
+        }
+      }
     } finally {
       if(dagPBOutBinaryStream != null){
         dagPBOutBinaryStream.close();
@@ -823,7 +841,7 @@ public class YARNRunner implements ClientProtocol {
     // Set up the ApplicationSubmissionContext
     ApplicationSubmissionContext appContext = Records
         .newRecord(ApplicationSubmissionContext.class);
-    
+
     appContext.setApplicationId(applicationId);                // ApplicationId
     appContext.setResource(capability);                        // resource
     appContext.setQueue(                                       // Queue name
@@ -870,12 +888,12 @@ public class YARNRunner implements ClientProtocol {
     Configuration tezJobConf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
 
     // This will replace job.xml in the staging dir.
-    writeTezConf(jobSubmitDir, fs, tezJobConf);    
+    writeTezConf(jobSubmitDir, fs, tezJobConf);
 
     // FIXME set up job resources
     Map<String, LocalResource> jobLocalResources =
         createJobLocalResources(tezJobConf, jobSubmitDir);
-    
+
     // FIXME createDAG should take the tezConf as a parameter, instead of using
     // MR keys.
     DAG dag = createDAG(fs, jobId, jobConf, jobSubmitDir, ts,
@@ -898,36 +916,6 @@ public class YARNRunner implements ClientProtocol {
           || appMasterReport.getYarnApplicationState() == YarnApplicationState.KILLED) {
         throw new IOException("Failed to run job : " + diagnostics);
       }
-
-      if (LOG.isDebugEnabled()) {
-        while (true) {
-          appMasterReport = resMgrDelegate.getApplicationReport(applicationId);
-          diagnostics = (appMasterReport == null ? "application report is null"
-              : appMasterReport.getDiagnostics());
-          if (appMasterReport == null) {
-            throw new IOException("Failed to run job : " + diagnostics);
-          }
-          YarnApplicationState state = appMasterReport
-              .getYarnApplicationState();
-          if (state.equals(YarnApplicationState.FAILED)
-              || state.equals(YarnApplicationState.FINISHED)
-              || state.equals(YarnApplicationState.KILLED)) {
-            LOG.info("Job completed" + ", finalStatus="
-                + appMasterReport.getFinalApplicationStatus() + ", finalState="
-                + appMasterReport.getYarnApplicationState() + ", diagnostics="
-                + diagnostics);
-            break;
-          } else {
-            LOG.info("Job in progress" + ", finalStatus="
-                + appMasterReport.getFinalApplicationStatus() + ", finalState="
-                + appMasterReport.getYarnApplicationState());
-          }
-          try {
-            Thread.sleep(1000);
-          } catch (InterruptedException e) {
-          }
-        }
-      }
     } catch (YarnRemoteException e) {
       throw new IOException(e);
     }
@@ -1089,7 +1077,7 @@ public class YARNRunner implements ClientProtocol {
                envConf + " config settings.");
     }
   }
-  
+
   @SuppressWarnings("deprecation")
   private String getMapJavaOpts(Configuration jobConf) {
     // follows pattern from YARN MapReduceChildJVM.java
@@ -1097,9 +1085,9 @@ public class YARNRunner implements ClientProtocol {
     adminOpts = jobConf.get(
         MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,
         MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
-    
+
     String userOpts = "";
-    userOpts = 
+    userOpts =
         jobConf.get(
             MRJobConfig.MAP_JAVA_OPTS, // same as JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS
                 jobConf.get(
@@ -1109,23 +1097,23 @@ public class YARNRunner implements ClientProtocol {
     return adminOpts.trim() + " " + userOpts.trim() + " "
     + getLog4jCmdLineProperties(jobConf, true);
   }
-  
+
   @SuppressWarnings("deprecation")
   private String getReduceJavaOpts(Configuration jobConf) {
-    // follows pattern from YARN MapReduceChildJVM.java 
+    // follows pattern from YARN MapReduceChildJVM.java
     String adminOpts = "";
     adminOpts = jobConf.get(
         MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,
         MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
-    
+
     String userOpts = "";
-    userOpts = 
+    userOpts =
         jobConf.get(
-            MRJobConfig.REDUCE_JAVA_OPTS, // same as JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS 
+            MRJobConfig.REDUCE_JAVA_OPTS, // same as JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS
                 jobConf.get(
                     JobConf.MAPRED_TASK_JAVA_OPTS,
                     JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
-    
+
     return adminOpts.trim() + " " + userOpts.trim() + " "
         + getLog4jCmdLineProperties(jobConf, false);
   }
@@ -1144,7 +1132,7 @@ public class YARNRunner implements ClientProtocol {
   /**
    * Add the JVM system properties necessary to configure
    * {@link ContainerLogAppender}.
-   * 
+   *
    * @param logLevel
    *          the desired log level (eg INFO/WARN/DEBUG)
    * @param vargs
@@ -1163,4 +1151,4 @@ public class YARNRunner implements ClientProtocol {
     vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
   }
 
-}
\ No newline at end of file
+}


[48/50] [abbrv] git commit: TEZ-177. Change Tez to use renamed YARN exceptions post YARN-635. (sseth)

Posted by ss...@apache.org.
TEZ-177. Change Tez to use renamed YARN exceptions post YARN-635.
(sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/114b12a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/114b12a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/114b12a9

Branch: refs/heads/master
Commit: 114b12a96f1d86fb3c3f01b458dd5b0a69133db6
Parents: 9feab05
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jun 3 22:04:12 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jun 3 22:04:12 2013 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/tez/client/TezClient.java |    4 +-
 .../hadoop/mapred/MRVertexOutputCommitter.java     |    4 +-
 .../java/org/apache/tez/dag/app/DAGAppMaster.java  |    6 ++--
 .../tez/dag/app/TaskAttemptListenerImpTezDag.java  |    4 +-
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java      |    4 +-
 .../dag/app/launcher/ContainerLauncherImpl.java    |    4 +-
 .../org/apache/tez/dag/app/rm/TaskScheduler.java   |   12 ++++----
 .../tez/dag/app/rm/TaskSchedulerEventHandler.java  |    6 ++--
 .../dag/app/rm/container/AMContainerHelpers.java   |    4 +-
 .../org/apache/tez/dag/app/rm/node/AMNodeMap.java  |    4 +-
 .../tez/dag/app/speculate/DefaultSpeculator.java   |   10 +++---
 .../tez/dag/app/taskclean/TaskCleanerImpl.java     |    4 +-
 .../apache/tez/engine/runtime/RuntimeUtils.java    |    4 +-
 .../apache/tez/mapreduce/MiniMRRTezCluster.java    |    4 +-
 .../tez/mapreduce/ClientServiceDelegate.java       |    6 ++--
 .../apache/tez/mapreduce/ResourceMgrDelegate.java  |   22 +++++++-------
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |   14 +++++-----
 17 files changed, 58 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
index 73c97b3..ec1dc57 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.client.YarnClient;
 import org.apache.hadoop.yarn.client.YarnClientImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
@@ -52,7 +52,7 @@ public class TezClient {
       String host = appReport.getHost();
       int port = appReport.getRpcPort();
       return getDAGClient(host, port);
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new TezException(e);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
index 171c244..b4949c0 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.client.VertexStatus;
 import org.apache.tez.dag.api.committer.VertexContext;
@@ -80,7 +80,7 @@ public class MRVertexOutputCommitter extends VertexOutputCommitter {
             .getOutputFormatClass(), conf);
         committer = outputFormat.getOutputCommitter(taskContext);
       } catch (Exception e) {
-        throw new YarnException(e);
+        throw new YarnRuntimeException(e);
       }
     } else {
       committer = ReflectionUtils.newInstance(conf.getClass(

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index a4ca97e..96f62a8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -48,7 +48,7 @@ import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.ClusterInfo;
 import org.apache.hadoop.yarn.SystemClock;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -579,7 +579,7 @@ public class DAGAppMaster extends CompositeService {
         }
       }
     } catch (IOException e) {
-      throw new YarnException(e);
+      throw new YarnRuntimeException(e);
     }
   }
 
@@ -827,7 +827,7 @@ public class DAGAppMaster extends CompositeService {
     @Override
     public Map<ApplicationAccessType, String> getApplicationACLs() {
       if (getServiceState() != STATE.STARTED) {
-        throw new YarnException(
+        throw new YarnRuntimeException(
             "Cannot get ApplicationACLs before all services have started");
       }
       return taskSchedulerEventHandler.getApplicationAcls();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 708b33e..14e3144 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.authorize.PolicyProvider;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.tez.common.ContainerContext;
@@ -129,7 +129,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
       server.start();
       this.address = NetUtils.getConnectAddress(server);
     } catch (IOException e) {
-      throw new YarnException(e);
+      throw new YarnRuntimeException(e);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 94e5161..0b6b23a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -583,7 +583,7 @@ public class TaskAttemptImpl implements TaskAttempt,
     case SUCCEEDED:
       return TaskAttemptState.SUCCEEDED;
     default:
-      throw new YarnException("Attempt to convert invalid "
+      throw new YarnRuntimeException("Attempt to convert invalid "
           + "stateMachineTaskAttemptState to externalTaskAttemptState: "
           + smState);
     }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index b71af02..5271bae 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.api.ContainerManager;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
@@ -426,7 +426,7 @@ public class ContainerLauncherImpl extends AbstractService implements
     try {
       eventQueue.put(event);
     } catch (InterruptedException e) {
-      throw new YarnException(e);
+      throw new YarnRuntimeException(e);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
index d081d2a..58795d7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
@@ -30,7 +30,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.client.AMRMClient.StoredContainerRequest;
 import org.apache.hadoop.yarn.client.AMRMClientAsync;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.hadoop.yarn.util.RackResolver;
 import org.apache.tez.dag.api.TezException;
@@ -196,12 +196,12 @@ public class TaskScheduler extends AbstractService
                                       response.getMinimumResourceCapability(),
                                       response.getMaximumResourceCapability(),
                                       response.getApplicationACLs());
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       LOG.error("Yarn Exception while registering", e);
-      throw new YarnException(e);
+      throw new YarnRuntimeException(e);
     } catch (IOException e) {
       LOG.error("IO Exception while registering", e);
-      throw new YarnException(e);
+      throw new YarnRuntimeException(e);
     }
   }
   
@@ -223,7 +223,7 @@ public class TaskScheduler extends AbstractService
       // to get our lock.
       amRmClient.stop();
       super.stop();
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       LOG.error("Yarn Exception while unregistering ", e);
       throw new TezException(e);
     } catch (IOException e) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index e29db1c..7d55bec 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -27,7 +27,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -119,7 +119,7 @@ public class TaskSchedulerEventHandler extends AbstractService
         handleTASucceeded(event);
         break;
       default:
-        throw new YarnException("Unexecpted TA_ENDED state: " + event.getState()); 
+        throw new YarnRuntimeException("Unexecpted TA_ENDED state: " + event.getState()); 
       }
       break;
     case S_CONTAINER_DEALLOCATE:
@@ -152,7 +152,7 @@ public class TaskSchedulerEventHandler extends AbstractService
     try {
       eventQueue.put(event);
     } catch (InterruptedException e) {
-      throw new YarnException(e);
+      throw new YarnRuntimeException(e);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 6156e33..9abff17 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -129,7 +129,7 @@ public class AMContainerHelpers {
           ShuffleHandler.serializeServiceData(jobToken));
 
     } catch (IOException e) {
-      throw new YarnException(e);
+      throw new YarnRuntimeException(e);
     }
 
     // Construct the actual Container

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
index a9256aa..efbdb6b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
@@ -26,7 +26,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -79,7 +79,7 @@ public class AMNodeMap extends AbstractService implements
         ", maxTaskFailuresPerNode: " + maxTaskFailuresPerNode);
 
     if (blacklistDisablePercent < -1 || blacklistDisablePercent > 100) {
-      throw new YarnException("Invalid blacklistDisablePercent: "
+      throw new YarnRuntimeException("Invalid blacklistDisablePercent: "
           + blacklistDisablePercent
           + ". Should be an integer between 0 and 100 or -1 to disabled");
     }    

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
index 79ccb8e..c5bb936 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
@@ -35,7 +35,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
@@ -127,16 +127,16 @@ public class DefaultSpeculator extends AbstractService implements
       estimator.contextualize(conf, context);
     } catch (InstantiationException ex) {
       LOG.error("Can't make a speculation runtime extimator", ex);
-      throw new YarnException(ex);
+      throw new YarnRuntimeException(ex);
     } catch (IllegalAccessException ex) {
       LOG.error("Can't make a speculation runtime extimator", ex);
-      throw new YarnException(ex);
+      throw new YarnRuntimeException(ex);
     } catch (InvocationTargetException ex) {
       LOG.error("Can't make a speculation runtime extimator", ex);
-      throw new YarnException(ex);
+      throw new YarnRuntimeException(ex);
     } catch (NoSuchMethodException ex) {
       LOG.error("Can't make a speculation runtime extimator", ex);
-      throw new YarnException(ex);
+      throw new YarnRuntimeException(ex);
     }
     
   return estimator;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanerImpl.java
index 4e6f897..1590cda 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanerImpl.java
@@ -27,7 +27,7 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.tez.dag.app.AppContext;
 
@@ -114,7 +114,7 @@ public class TaskCleanerImpl extends AbstractService implements TaskCleaner {
     try {
       eventQueue.put(event);
     } catch (InterruptedException e) {
-      throw new YarnException(e);
+      throw new YarnRuntimeException(e);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
index 1946a62..0dbda40 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
@@ -24,7 +24,7 @@ import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.tez.common.InputSpec;
 import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.TezEngineTaskContext;
@@ -119,7 +119,7 @@ public class RuntimeUtils {
       }
       t = createRuntime(taskContext, processor, inputs, outputs);
     } catch (ClassNotFoundException e) {
-      throw new YarnException("Unable to initialize RuntimeTask, context="
+      throw new YarnRuntimeException("Unable to initialize RuntimeTask, context="
           + taskContext, e);
     }
     return t;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
index 41a1cf3..bbf5e68 100644
--- a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
+++ b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.ShuffleHandler;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
 import org.apache.hadoop.util.JarFinder;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
@@ -100,7 +100,7 @@ public class MiniMRRTezCluster extends MiniYARNCluster {
       Path doneDirPath = fc.makeQualified(new Path(doneDir));
       fc.mkdir(doneDirPath, null, true);
     } catch (IOException e) {
-      throw new YarnException("Could not create staging directory. ", e);
+      throw new YarnRuntimeException("Could not create staging directory. ", e);
     }
     conf.set(MRConfig.MASTER_ADDRESS, "test");
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
index 1335a44..88fafbf 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.mapreduce.TaskCompletionEvent;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.tez.client.TezClient;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezRemoteException;
@@ -155,7 +155,7 @@ public class ClientServiceDelegate {
 
   public LogParams getLogFilePath(JobID oldJobID,
       TaskAttemptID oldTaskAttemptID)
-      throws YarnRemoteException, IOException {
+      throws YarnException, IOException {
     // FIXME logs for an attempt?
     throw new UnsupportedOperationException();
   }
@@ -165,7 +165,7 @@ public class ClientServiceDelegate {
       TypeConverter.toYarn(oldJobID);
     try {
       return rm.getApplicationReport(jobId.getAppId());
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
index 2561b56..8563a09 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
 import org.apache.hadoop.yarn.client.YarnClientImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.ProtoUtils;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
@@ -67,7 +67,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
       InterruptedException {
     try {
       return TypeConverter.fromYarnNodes(super.getNodeReports());
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
   }
@@ -75,7 +75,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
   public JobStatus[] getAllJobs() throws IOException, InterruptedException {
     try {
       return TypeConverter.fromYarnApps(super.getApplicationList(), this.conf);
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
   }
@@ -92,7 +92,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
     YarnClusterMetrics metrics;
     try {
       metrics = super.getYarnClusterMetrics();
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
     ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, 
@@ -107,7 +107,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
     try {
       return ProtoUtils.convertFromProtoFormat(
         super.getRMDelegationToken(renewer), rmAddress);
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
   }
@@ -119,7 +119,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
   public JobID getNewJobID() throws IOException, InterruptedException {
     try {
       this.application = super.getNewApplication();
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
     this.applicationId = this.application.getApplicationId();
@@ -131,7 +131,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
     try {
       return TypeConverter.fromYarn(
           super.getQueueInfo(queueName), this.conf);
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
   }
@@ -141,7 +141,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
     try {
       return TypeConverter.fromYarnQueueUserAclsInfo(super
         .getQueueAclsInfo());
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
   }
@@ -149,7 +149,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
   public QueueInfo[] getQueues() throws IOException, InterruptedException {
     try {
       return TypeConverter.fromYarnQueueInfo(super.getAllQueues(), this.conf);
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
   }
@@ -157,7 +157,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
   public QueueInfo[] getRootQueues() throws IOException, InterruptedException {
     try {
       return TypeConverter.fromYarnQueueInfo(super.getRootQueueInfos(), this.conf);
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
   }
@@ -167,7 +167,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
     try {
       return TypeConverter.fromYarnQueueInfo(super.getChildQueueInfos(parent),
         this.conf);
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 05b5411..592648e 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -72,7 +72,7 @@ import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
@@ -87,7 +87,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
@@ -888,7 +888,7 @@ public class YARNRunner implements ClientProtocol {
     try {
       ts.writeTokenStorageFile(applicationTokensFile, conf);
     } catch (IOException e) {
-      throw new YarnException(e);
+      throw new YarnRuntimeException(e);
     }
 
     FileSystem fs = FileSystem.get(conf);
@@ -924,7 +924,7 @@ public class YARNRunner implements ClientProtocol {
           || appMasterReport.getYarnApplicationState() == YarnApplicationState.KILLED) {
         throw new IOException("Failed to run job : " + diagnostics);
       }
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
 
@@ -1007,7 +1007,7 @@ public class YARNRunner implements ClientProtocol {
     if (status.getState() != JobStatus.State.RUNNING) {
       try {
         resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
-      } catch (YarnRemoteException e) {
+      } catch (YarnException e) {
         throw new IOException(e);
       }
       return;
@@ -1035,7 +1035,7 @@ public class YARNRunner implements ClientProtocol {
     if (status.getState() != JobStatus.State.KILLED) {
       try {
         resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
-      } catch (YarnRemoteException e) {
+      } catch (YarnException e) {
         throw new IOException(e);
       }
     }
@@ -1070,7 +1070,7 @@ public class YARNRunner implements ClientProtocol {
       throws IOException {
     try {
       return clientCache.getClient(jobID).getLogFilePath(jobID, taskAttemptID);
-    } catch (YarnRemoteException e) {
+    } catch (YarnException e) {
       throw new IOException(e);
     }
   }


[35/50] [abbrv] git commit: TEZ-164. Fix handling of completed, running, pending tasks in AMContainer. (sseth)

Posted by ss...@apache.org.
TEZ-164. Fix handling of completed, running, pending tasks in
AMContainer. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/9f0d4c3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/9f0d4c3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/9f0d4c3c

Branch: refs/heads/master
Commit: 9f0d4c3cb4a8b4b3b6ea89e2c0cb0dc85bc91545
Parents: a74b436
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed May 29 16:35:40 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed May 29 16:35:40 2013 -0700

----------------------------------------------------------------------
 .../tez/dag/app/rm/container/AMContainer.java      |    3 +-
 .../tez/dag/app/rm/container/AMContainerImpl.java  |  100 +++++++--------
 .../tez/dag/app/rm/container/TestAMContainer.java  |   93 ++++++-------
 3 files changed, 93 insertions(+), 103 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f0d4c3c/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
index 3c00620..e62d033 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
@@ -30,8 +30,7 @@ public interface AMContainer extends EventHandler<AMContainerEvent>{
   public AMContainerState getState();
   public ContainerId getContainerId();
   public Container getContainer();
-  //TODO Rename - CompletedTaskAttempts, ideally means FAILED / KILLED as well.
-  public List<TezTaskAttemptID> getCompletedTaskAttempts();
+  public List<TezTaskAttemptID> getAllTaskAttempts();
   public TezTaskAttemptID getRunningTaskAttempt();
   public List<TezTaskAttemptID> getQueuedTaskAttempts();
   

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f0d4c3c/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 51246dd..58e9d50 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -18,7 +18,6 @@
 
 package org.apache.tez.dag.app.rm.container;
 
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -48,13 +47,13 @@ import org.apache.tez.dag.app.TaskAttemptListener;
 import org.apache.tez.dag.app.dag.event.DiagnosableEvent;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminating;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventDiagnosticsUpdate;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventNodeFailed;
 import org.apache.tez.dag.app.rm.AMSchedulerEventContainerCompleted;
 import org.apache.tez.dag.app.rm.AMSchedulerEventDeallocateContainer;
 import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
 import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
 import org.apache.tez.dag.records.TezTaskAttemptID;
+//import org.apache.tez.dag.app.dag.event.TaskAttemptEventDiagnosticsUpdate;
 
 @SuppressWarnings("rawtypes")
 public class AMContainerImpl implements AMContainer {
@@ -222,10 +221,19 @@ public class AMContainerImpl implements AMContainer {
   }
 
   @Override
-  public List<TezTaskAttemptID> getCompletedTaskAttempts() {
+  public List<TezTaskAttemptID> getAllTaskAttempts() {
     readLock.lock();
     try {
-      return new ArrayList<TezTaskAttemptID>(this.completedAttempts);
+      List<TezTaskAttemptID> allAttempts = new LinkedList<TezTaskAttemptID>();
+      allAttempts.addAll(this.completedAttempts);
+      allAttempts.addAll(this.failedAssignments);
+      if (this.pendingAttempt != null) {
+        allAttempts.add(this.pendingAttempt);
+      }
+      if (this.runningAttempt != null) {
+        allAttempts.add(this.runningAttempt);
+      }
+      return allAttempts;
     } finally {
       readLock.unlock();
     }
@@ -359,6 +367,7 @@ public class AMContainerImpl implements AMContainer {
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       AMContainerEventAssignTA event = (AMContainerEventAssignTA) cEvent;
       container.inError = true;
+      container.registerFailedAttempt(event.getTaskAttemptId());
       container.maybeSendNodeFailureForFailedAssignment(event
           .getTaskAttemptId());
       container.sendTerminatedToTaskAttempt(event.getTaskAttemptId(),
@@ -378,7 +387,6 @@ public class AMContainerImpl implements AMContainer {
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       AMContainerEventCompleted event = (AMContainerEventCompleted)cEvent;
       container.sendCompletedToScheduler();
-      container.sendDiagUpdateOnContainerComplete(event);
       String diag = event.getContainerStatus().getDiagnostics();
       if (!(diag == null || diag.equals(""))) {
         LOG.info("Container " + container.getContainerId()
@@ -402,10 +410,11 @@ public class AMContainerImpl implements AMContainer {
     }
   }
 
-  protected static class NodeFailedAtAllocatedTransition implements
-      SingleArcTransition<AMContainerImpl, AMContainerEvent> {
+  protected static class NodeFailedAtAllocatedTransition extends
+      NodeFailedBaseTransition {
     @Override
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
+      super.transition(container, cEvent);
       container.sendCompletedToScheduler();
       container.deAllocate();
     }
@@ -414,6 +423,7 @@ public class AMContainerImpl implements AMContainer {
   protected static class ErrorTransition extends ErrorBaseTransition {
     @Override
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
+      super.transition(container, cEvent);
       container.sendCompletedToScheduler();
       container.deAllocate();
       LOG.info(
@@ -464,7 +474,7 @@ public class AMContainerImpl implements AMContainer {
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       if (container.pendingAttempt != null) {
         AMContainerEventLaunchFailed event = (AMContainerEventLaunchFailed) cEvent;
-        container.sendTerminatingToTA(container.pendingAttempt,
+        container.sendTerminatingToTaskAttempt(container.pendingAttempt,
             event.getMessage());
       }
       container.unregisterFromTAListener();
@@ -481,9 +491,10 @@ public class AMContainerImpl implements AMContainer {
         String errorMessage = getMessage(container, event);
         container.sendTerminatedToTaskAttempt(container.pendingAttempt,
             errorMessage);
+        container.registerFailedAttempt(container.pendingAttempt);
+        container.pendingAttempt = null;
         LOG.warn(errorMessage);
       }
-      container.sendDiagUpdateOnContainerComplete(event);
       container.unregisterFromTAListener();
       container.sendCompletedToScheduler();
       String diag = event.getContainerStatus().getDiagnostics();
@@ -507,7 +518,7 @@ public class AMContainerImpl implements AMContainer {
     @Override
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       if (container.pendingAttempt != null) {
-        container.sendTerminatingToTA(container.pendingAttempt,
+        container.sendTerminatingToTaskAttempt(container.pendingAttempt,
             getMessage(container, cEvent));
       }
       container.unregisterFromTAListener();
@@ -541,12 +552,14 @@ public class AMContainerImpl implements AMContainer {
       }
 
       if (container.pendingAttempt != null) {
+        // Will be null in COMPLETED state.
         container.sendNodeFailureToTA(container.pendingAttempt, errorMessage);
-        container.sendTerminatingToTA(container.pendingAttempt, "Node failure");
+        container.sendTerminatingToTaskAttempt(container.pendingAttempt, "Node failure");
       }
       if (container.runningAttempt != null) {
+        // Will be null in COMPLETED state.
         container.sendNodeFailureToTA(container.runningAttempt, errorMessage);
-        container.sendTerminatingToTA(container.runningAttempt, "Node failure");
+        container.sendTerminatingToTaskAttempt(container.runningAttempt, "Node failure");
       }
     }
   }
@@ -562,16 +575,17 @@ public class AMContainerImpl implements AMContainer {
   }
 
   protected static class ErrorAtLaunchingTransition
-      extends ErrorTransition {
+      extends ErrorBaseTransition {
     @Override
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       super.transition(container, cEvent);
       if (container.pendingAttempt != null) {
-        container.sendTerminatedToTaskAttempt(container.pendingAttempt,
+        container.sendTerminatingToTaskAttempt(container.pendingAttempt, 
             "Container " + container.getContainerId() +
                 " hit an invalid transition - " + cEvent.getType() + " at " +
                 container.getState());
       }
+      container.sendStopRequestToNM();
       container.unregisterFromTAListener();
     }
   }
@@ -625,7 +639,6 @@ public class AMContainerImpl implements AMContainer {
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       super.transition(container, cEvent);
       container.unregisterFromContainerListener();
-      container.sendDiagUpdateOnContainerComplete((AMContainerEventCompleted)cEvent);
     }
 
     @Override
@@ -707,6 +720,8 @@ public class AMContainerImpl implements AMContainer {
       container.sendTerminatedToTaskAttempt(container.runningAttempt,
           getMessage(container, event));
       container.unregisterAttemptFromListener(container.runningAttempt);
+      container.registerFailedAttempt(container.runningAttempt);
+      container.runningAttempt = null;
       super.transition(container, cEvent);
     }
   }
@@ -716,7 +731,7 @@ public class AMContainerImpl implements AMContainer {
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
 
       container.unregisterAttemptFromListener(container.runningAttempt);
-      container.sendTerminatingToTA(container.runningAttempt,
+      container.sendTerminatingToTaskAttempt(container.runningAttempt,
           " Container" + container.getContainerId() +
               " received a STOP_REQUEST");
       super.transition(container, cEvent);
@@ -749,7 +764,7 @@ public class AMContainerImpl implements AMContainer {
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       super.transition(container, cEvent);
       container.unregisterAttemptFromListener(container.runningAttempt);
-      container.sendTerminatedToTaskAttempt(container.runningAttempt,
+      container.sendTerminatingToTaskAttempt(container.runningAttempt,
           "Container " + container.getContainerId() +
               " hit an invalid transition - " + cEvent.getType() + " at " +
               container.getState());
@@ -766,8 +781,8 @@ public class AMContainerImpl implements AMContainer {
           " cannot be allocated to container: " + container.getContainerId() +
           " in " + container.getState() + " state";
       container.maybeSendNodeFailureForFailedAssignment(event.getTaskAttemptId());
-      container.sendTerminatingToTA(event.getTaskAttemptId(), errorMessage);
-      container.registerFailedTAAssignment(event.getTaskAttemptId());
+      container.sendTerminatingToTaskAttempt(event.getTaskAttemptId(), errorMessage);
+      container.registerFailedAttempt(event.getTaskAttemptId());
     }
   }
   
@@ -788,14 +803,18 @@ public class AMContainerImpl implements AMContainer {
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       AMContainerEventCompleted event = (AMContainerEventCompleted) cEvent;
       String diag = event.getContainerStatus().getDiagnostics();
+      for (TezTaskAttemptID taId : container.failedAssignments) {
+        container.sendTerminatedToTaskAttempt(taId, diag);
+      }
       if (container.pendingAttempt != null) {
         container.sendTerminatedToTaskAttempt(container.pendingAttempt, diag);
+        container.registerFailedAttempt(container.pendingAttempt);
+        container.pendingAttempt = null;
       }
       if (container.runningAttempt != null) {
         container.sendTerminatedToTaskAttempt(container.runningAttempt, diag);
-      }
-      for (TezTaskAttemptID taId : container.failedAssignments) {
-        container.sendTerminatedToTaskAttempt(taId, diag);
+        container.registerFailedAttempt(container.runningAttempt);
+        container.runningAttempt = null;
       }
       if (!(diag == null || diag.equals(""))) {
         LOG.info("Container " + container.getContainerId()
@@ -822,10 +841,9 @@ public class AMContainerImpl implements AMContainer {
   }
 
   protected static class ErrorAtNMStopRequestedTransition
-      extends ErrorAtStoppingTransition {
+      extends ErrorBaseTransition {
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       super.transition(container, cEvent);
-      container.deAllocate();
     }
   }
 
@@ -833,15 +851,6 @@ public class AMContainerImpl implements AMContainer {
       extends ErrorBaseTransition {
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       super.transition(container, cEvent);
-      if (container.pendingAttempt != null) {
-        container.sendTerminatedToTaskAttempt(container.pendingAttempt, null);
-      }
-      if (container.runningAttempt != null) {
-        container.sendTerminatedToTaskAttempt(container.runningAttempt, null);
-      }
-      for (TezTaskAttemptID taId : container.failedAssignments) {
-        container.sendTerminatedToTaskAttempt(taId, null);
-      }
       container.sendCompletedToScheduler();
     }
   }
@@ -873,7 +882,7 @@ public class AMContainerImpl implements AMContainer {
       container.maybeSendNodeFailureForFailedAssignment(event.getTaskAttemptId());
       container.sendTerminatedToTaskAttempt(event.getTaskAttemptId(),
           errorMessage);
-      container.registerFailedTAAssignment(event.getTaskAttemptId());
+      container.registerFailedAttempt(event.getTaskAttemptId());
     }
   }
 
@@ -886,17 +895,16 @@ public class AMContainerImpl implements AMContainer {
         ". Attempts: " + currentTaId + ", " + event.getTaskAttemptId() +
         ". Current state: " + this.getState();
     this.maybeSendNodeFailureForFailedAssignment(event.getTaskAttemptId());
-    this.sendTerminatingToTA(event.getTaskAttemptId(), errorMessage);
-    this.sendTerminatingToTA(currentTaId, errorMessage);
-    this.registerFailedTAAssignment(event.getTaskAttemptId());
+    this.sendTerminatingToTaskAttempt(event.getTaskAttemptId(), errorMessage);
+    this.sendTerminatingToTaskAttempt(currentTaId, errorMessage);
+    this.registerFailedAttempt(event.getTaskAttemptId());
     LOG.warn(errorMessage);
     this.sendStopRequestToNM();
     this.unregisterFromTAListener();
     this.unregisterFromContainerListener();
   }
 
-
-  protected void registerFailedTAAssignment(TezTaskAttemptID taId) {
+  protected void registerFailedAttempt(TezTaskAttemptID taId) {
     failedAssignments.add(taId);
   }
   
@@ -908,23 +916,13 @@ public class AMContainerImpl implements AMContainer {
     sendEvent(new AMSchedulerEventContainerCompleted(containerId));
   }
 
-  protected void sendDiagUpdateOnContainerComplete(
-      AMContainerEventCompleted cEvent) {
-    String diag = cEvent.getContainerStatus().getDiagnostics();
-    if (pendingAttempt != null) {
-      sendEvent(new TaskAttemptEventDiagnosticsUpdate(pendingAttempt, diag));
-    }
-    if (runningAttempt != null) {
-      sendEvent(new TaskAttemptEventDiagnosticsUpdate(runningAttempt, diag));
-    }
-  }
-
   protected void sendTerminatedToTaskAttempt(
       TezTaskAttemptID taId, String message) {
     sendEvent(new TaskAttemptEventContainerTerminated(taId, message));
   }
 
-  protected void sendTerminatingToTA(TezTaskAttemptID taId, String message) {
+  protected void sendTerminatingToTaskAttempt(TezTaskAttemptID taId,
+      String message) {
     sendEvent(new TaskAttemptEventContainerTerminating(taId, message));
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f0d4c3c/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index 574a2c5..4541a2e 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -67,7 +67,6 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
 import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 
@@ -113,7 +112,7 @@ public class TestAMContainer {
         .getTaskAttemptId());
     assertEquals(wc.taskAttemptID, wc.amContainer.getRunningTaskAttempt());
     assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
-    
+
     wc.taskAttemptSucceeded(wc.taskAttemptID);
     wc.verifyState(AMContainerState.IDLE);
     wc.verifyNoOutgoingEvents();
@@ -127,7 +126,7 @@ public class TestAMContainer {
     verify(wc.tal).unregisterRunningContainer(wc.containerID);
     verify(wc.chh).unregister(wc.containerID);
 
-    assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+    assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
     assertFalse(wc.amContainer.isInErrorState());
   }
   
@@ -183,7 +182,7 @@ public class TestAMContainer {
     verify(wc.tal).unregisterRunningContainer(wc.containerID);
     verify(wc.chh).unregister(wc.containerID);
     
-    assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+    assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
     assertFalse(wc.amContainer.isInErrorState());
   }
   
@@ -219,7 +218,7 @@ public class TestAMContainer {
 
     assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
     assertNull(wc.amContainer.getRunningTaskAttempt());
-    assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+    assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
     assertFalse(wc.amContainer.isInErrorState());
   }
   
@@ -258,7 +257,7 @@ public class TestAMContainer {
 
     assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
     assertNull(wc.amContainer.getRunningTaskAttempt());
-    assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+    assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
     assertFalse(wc.amContainer.isInErrorState());
   }
   
@@ -297,8 +296,8 @@ public class TestAMContainer {
         AMSchedulerEventType.S_CONTAINER_COMPLETED);
     
     assertNull(wc.amContainer.getRunningTaskAttempt());
-//    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
-//    assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+    assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
   }
   
   @SuppressWarnings("rawtypes")
@@ -306,16 +305,16 @@ public class TestAMContainer {
   public void testAllocationAtRunning() {
     WrappedContainer wc = new WrappedContainer();
     List<Event> outgoingEvents;
-    
+
     wc.launchContainer();
     wc.containerLaunched();
     wc.assignTaskAttempt(wc.taskAttemptID);
     wc.pullTaskToRun();
     wc.verifyState(AMContainerState.RUNNING);
-    
+
     TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
     wc.assignTaskAttempt(taID2);
-    
+
     wc.verifyState(AMContainerState.STOP_REQUESTED);
     verify(wc.tal).unregisterRunningContainer(wc.containerID);
     verify(wc.chh).unregister(wc.containerID);
@@ -326,7 +325,7 @@ public class TestAMContainer {
         TaskAttemptEventType.TA_CONTAINER_TERMINATING,
         TaskAttemptEventType.TA_CONTAINER_TERMINATING);
     assertTrue(wc.amContainer.isInErrorState());
-    
+
     wc.nmStopSent();
     wc.containerCompleted();
     // 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
@@ -335,10 +334,10 @@ public class TestAMContainer {
         TaskAttemptEventType.TA_CONTAINER_TERMINATED,
         TaskAttemptEventType.TA_CONTAINER_TERMINATED,
         AMSchedulerEventType.S_CONTAINER_COMPLETED);
-    
-//  assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
-//  assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
-//  assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+    assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
   }
   
   @SuppressWarnings("rawtypes")
@@ -346,15 +345,15 @@ public class TestAMContainer {
   public void testMultipleAllocationsAtLaunching() {
     WrappedContainer wc = new WrappedContainer();
     List<Event> outgoingEvents;
-    
+
     wc.launchContainer();
     wc.assignTaskAttempt(wc.taskAttemptID);
     wc.pullTaskToRun();
     wc.verifyState(AMContainerState.LAUNCHING);
-    
+
     TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
     wc.assignTaskAttempt(taID2);
-    
+
     wc.verifyState(AMContainerState.STOP_REQUESTED);
     verify(wc.tal).unregisterRunningContainer(wc.containerID);
     verify(wc.chh).unregister(wc.containerID);
@@ -365,7 +364,7 @@ public class TestAMContainer {
         TaskAttemptEventType.TA_CONTAINER_TERMINATING,
         TaskAttemptEventType.TA_CONTAINER_TERMINATING);
     assertTrue(wc.amContainer.isInErrorState());
-    
+
     wc.nmStopSent();
     wc.containerCompleted();
     // 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
@@ -374,10 +373,10 @@ public class TestAMContainer {
         TaskAttemptEventType.TA_CONTAINER_TERMINATED,
         TaskAttemptEventType.TA_CONTAINER_TERMINATED,
         AMSchedulerEventType.S_CONTAINER_COMPLETED);
-    
-//  assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
-//  assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
-//  assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+    assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
   }
 
   @SuppressWarnings("rawtypes")
@@ -385,13 +384,13 @@ public class TestAMContainer {
   public void testContainerTimedOutAtRunning() {
     WrappedContainer wc = new WrappedContainer();
     List<Event> outgoingEvents;
-    
+
     wc.launchContainer();
     wc.containerLaunched();
     wc.assignTaskAttempt(wc.taskAttemptID);
     wc.pullTaskToRun();
     wc.verifyState(AMContainerState.RUNNING);
-    
+
     wc.containerTimedOut();
     wc.verifyState(AMContainerState.STOP_REQUESTED);
     verify(wc.tal).unregisterRunningContainer(wc.containerID);
@@ -402,18 +401,18 @@ public class TestAMContainer {
         TaskAttemptEventType.TA_CONTAINER_TERMINATING,
         NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
     // TODO Should this be an RM DE-ALLOCATE instead ?
-    
+
     wc.containerCompleted();
     outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
     verifyUnOrderedOutgoingEventTypes(outgoingEvents,
         TaskAttemptEventType.TA_CONTAINER_TERMINATED,
         AMSchedulerEventType.S_CONTAINER_COMPLETED);
-    
+
     assertFalse(wc.amContainer.isInErrorState());
-    
-//  assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
-//  assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
-//  assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+    assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
   }
 
   @SuppressWarnings("rawtypes")
@@ -461,7 +460,6 @@ public class TestAMContainer {
     assertFalse(wc.amContainer.isInErrorState());
   }
 
-  @Ignore
   @SuppressWarnings("rawtypes")
   @Test
   // Verify that incoming NM launched events to COMPLETED containers are
@@ -484,7 +482,6 @@ public class TestAMContainer {
     verifyUnOrderedOutgoingEventTypes(outgoingEvents,
         AMSchedulerEventType.S_CONTAINER_COMPLETED,
         TaskAttemptEventType.TA_CONTAINER_TERMINATED);
-    // TODO Failing because of an extra diagnostic event.
     
     assertFalse(wc.amContainer.isInErrorState());
     
@@ -495,7 +492,6 @@ public class TestAMContainer {
     assertFalse(wc.amContainer.isInErrorState());
   }
 
-  @Ignore
   @SuppressWarnings("rawtypes")
   @Test
   public void testContainerCompletedAtIdle() {
@@ -519,7 +515,6 @@ public class TestAMContainer {
     verifyUnOrderedOutgoingEventTypes(outgoingEvents,
         AMSchedulerEventType.S_CONTAINER_COMPLETED,
         TaskAttemptEventType.TA_CONTAINER_TERMINATED);
-    // TODO Failing because of two extra diagnostic event.
     
     assertFalse(wc.amContainer.isInErrorState());
     
@@ -531,8 +526,7 @@ public class TestAMContainer {
     
     assertFalse(wc.amContainer.isInErrorState());
   }
-  
-  @Ignore
+
   @SuppressWarnings("rawtypes")
   @Test
   public void testContainerCompletedAtRunning() {
@@ -557,7 +551,6 @@ public class TestAMContainer {
     verifyUnOrderedOutgoingEventTypes(outgoingEvents,
         AMSchedulerEventType.S_CONTAINER_COMPLETED,
         TaskAttemptEventType.TA_CONTAINER_TERMINATED);
-    // TODO Failing because of two extra diagnostic event.
     
     assertFalse(wc.amContainer.isInErrorState());
     
@@ -693,7 +686,7 @@ public class TestAMContainer {
     
     assertNull(wc.amContainer.getRunningTaskAttempt());
     assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
-    assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size());
+    assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
   }
   
   @SuppressWarnings("rawtypes")
@@ -701,18 +694,18 @@ public class TestAMContainer {
   public void testNodeFailedAtRunningMultipleAttempts() {
     WrappedContainer wc = new WrappedContainer();
     List<Event> outgoingEvents;
-    
+
     wc.launchContainer();
     wc.containerLaunched();
     wc.assignTaskAttempt(wc.taskAttemptID);
     wc.pullTaskToRun();
     wc.taskAttemptSucceeded(wc.taskAttemptID);
-    
+
     TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
     wc.assignTaskAttempt(taID2);
     wc.pullTaskToRun();
     wc.verifyState(AMContainerState.RUNNING);
-    
+
     wc.nodeFailed();
     // Expecting a complete event from the RM
     wc.verifyState(AMContainerState.STOPPING);
@@ -722,14 +715,14 @@ public class TestAMContainer {
         TaskAttemptEventType.TA_NODE_FAILED,
         TaskAttemptEventType.TA_CONTAINER_TERMINATING,
         AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
-    
+
     for (Event event : outgoingEvents) {
       if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
         TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
         assertEquals("nodeFailed", nfEvent.getDiagnosticInfo());
       }
     }
-    
+
     wc.containerCompleted();
     outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
     verifyUnOrderedOutgoingEventTypes(outgoingEvents,
@@ -737,11 +730,11 @@ public class TestAMContainer {
         AMSchedulerEventType.S_CONTAINER_COMPLETED);
 
     assertFalse(wc.amContainer.isInErrorState());
-//    assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO. Set/Unset properly.
-//    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
-//    assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+    assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
   }
-  
+
   @SuppressWarnings("rawtypes")
   @Test
   public void testNodeFailedAtCompletedMultipleSuccessfulTAs() {
@@ -771,7 +764,7 @@ public class TestAMContainer {
     
     assertNull(wc.amContainer.getRunningTaskAttempt());
     assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
-    assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size());
+    assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
   }
 
   @SuppressWarnings("rawtypes")


[39/50] [abbrv] git commit: TEZ-173. Fix ContainerLauncher to use containerToken in the startContainer call [YARN-684]. (sseth)

Posted by ss...@apache.org.
TEZ-173. Fix ContainerLauncher to use containerToken in the
startContainer call [YARN-684]. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a6791898
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a6791898
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a6791898

Branch: refs/heads/master
Commit: a6791898ee72ac1e23aca93b458000e2207bfb35
Parents: 5d0b8ad
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu May 30 22:24:14 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu May 30 22:24:14 2013 -0700

----------------------------------------------------------------------
 .../dag/app/launcher/ContainerLauncherImpl.java    |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a6791898/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index f1a16b8..5b23c75 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -160,7 +160,7 @@ public class ContainerLauncherImpl extends AbstractService implements
         // Now launch the actual container
         StartContainerRequest startRequest = Records
           .newRecord(StartContainerRequest.class);
-        startRequest.setContainer(event.getContainer());
+        startRequest.setContainerToken(event.getContainerToken());
         startRequest.setContainerLaunchContext(containerLaunchContext);
         StartContainerResponse response = proxy.startContainer(startRequest);
 


[11/50] [abbrv] git commit: TEZ-31: implement DAG.verify() to validate structural properties of dag created through tez-dag-api

Posted by ss...@apache.org.
TEZ-31: implement DAG.verify() to validate structural properties of dag created through tez-dag-api


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/645c470e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/645c470e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/645c470e

Branch: refs/heads/master
Commit: 645c470eeb1c4476437498fed639c3c20f57c672
Parents: d660948
Author: mikelid <mi...@microsoft.com>
Authored: Tue May 14 17:13:14 2013 -0700
Committer: mikelid <mi...@microsoft.com>
Committed: Tue May 14 17:13:14 2013 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/tez/dag/api/DAG.java  |  149 +++++++++++-
 .../java/org/apache/tez/dag/api/TestDAGVerify.java |  194 +++++++++++++++
 2 files changed, 340 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/645c470e/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 171921d..300ef31 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Stack;
 
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -87,13 +88,155 @@ public class DAG { // FIXME rename to Topology
     this.name = name;
   }
   
-  public void verify() throws TezException { // FIXME better exception
-    //FIXME are task resources compulsory or will the DAG AM put in a default
-    //for each vertex if not specified?
+  // AnnotatedVertex is used by verify() 
+  private class AnnotatedVertex {
+    Vertex v;
+  
+    int index; //for Tarjan's algorithm    
+    int lowlink; //for Tarjan's algorithm
+    boolean onstack; //for Tarjan's algorithm 
+
+    int inDegree;
+    int outDegree;
+    
+    private AnnotatedVertex(Vertex v){
+       this.v = v;
+       index = -1;
+       lowlink = -1;
+       inDegree = 0;
+       outDegree = 0;
+    }
   }
+  
+  // verify()
+  // 
+  // Default rules
+  //   Illegal:
+  //     - duplicate vertex id
+  //     - cycles
+  //
+  //   Ok:
+  //     - orphaned vertex.  Occurs in map-only
+  //     - islands.  Occurs if job has unrelated workflows.
+  //
+  //   Not yet categorized:
+  //     - orphaned vertex in DAG of >1 vertex.  Could be unrelated map-only job.
+  //     - v1->v2 via two edges.  perhaps some self-join job would use this?
+  //
+  // "restricted" mode: 
+  //   In short term, the supported DAGs are limited. Call with restricted=true for these verifications.  
+  //   Illegal: 
+  //     - any vertex with more than one input or output edge. (n-ary input, n-ary merge) 
+  public void verify() throws IllegalStateException {
+    verify(true);
+  }
+  
+  public void verify(boolean restricted) throws IllegalStateException  { 
+    Map<Vertex, List<Edge>> edgeMap = new HashMap<Vertex, List<Edge>>();
+    for(Edge e : edges){
+      Vertex inputVertex = e.getInputVertex();
+      List<Edge> edgeList = edgeMap.get(inputVertex);
+      if(edgeList == null){
+        edgeList = new ArrayList<Edge>();
+        edgeMap.put(inputVertex, edgeList);
+      }
+      edgeList.add(e);
+    }
     
+    // check for duplicate vertex names, and prepare for cycle detection
+    Map<String, AnnotatedVertex> vertexMap = new HashMap<String, AnnotatedVertex>();
+    for(Vertex v : vertices){
+      if(vertexMap.containsKey(v.getVertexName())){
+         throw new IllegalStateException("DAG contains multiple vertices with name: " + v.getVertexName());
+      }
+      vertexMap.put(v.getVertexName(), new AnnotatedVertex(v));
+    }
+    
+    detectCycles(edgeMap, vertexMap);
+    
+    if(restricted){
+      for(Edge e : edges){
+        vertexMap.get(e.getInputVertex().getVertexName()).outDegree++;
+        vertexMap.get(e.getOutputVertex().getVertexName()).inDegree++;
+      }
+      for(AnnotatedVertex av: vertexMap.values()){
+        if(av.inDegree > 1){
+          throw new IllegalStateException("Vertex has inDegree>1: " + av.v.getVertexName());
+        }
+        if(av.outDegree > 1){
+          throw new IllegalStateException("Vertex has outDegree>1: " + av.v.getVertexName());
+        }
+      }
+    }
+  }
+  
+  // Adaptation of Tarjan's algorithm for connected components.
+  // http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
+  private void detectCycles(Map<Vertex, List<Edge>> edgeMap, Map<String, AnnotatedVertex> vertexMap) 
+      throws IllegalStateException{
+    Integer nextIndex = 0; // boxed integer so it is passed by reference.
+    Stack<AnnotatedVertex> stack = new Stack<DAG.AnnotatedVertex>();
+    for(AnnotatedVertex av: vertexMap.values()){
+      if(av.index == -1){
+        assert stack.empty();
+        strongConnect(av, vertexMap, edgeMap, stack, nextIndex);
+      }
+    }
+  }
+
+  // part of Tarjan's algorithm for connected components.
+  // http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
+  private void strongConnect(
+          AnnotatedVertex av, 
+          Map<String, AnnotatedVertex> vertexMap, 
+          Map<Vertex, List<Edge>> edgeMap, 
+          Stack<AnnotatedVertex> stack, Integer nextIndex) throws IllegalStateException{
+    av.index = nextIndex;
+    av.lowlink = nextIndex;
+    nextIndex++;
+    stack.push(av);
+    av.onstack = true;
+    
+    List<Edge> edges = edgeMap.get(av.v);
+    if(edges != null){
+      for(Edge e : edgeMap.get(av.v)){
+        AnnotatedVertex outVertex = vertexMap.get(e.getOutputVertex().getVertexName());
+        if(outVertex.index == -1){
+          strongConnect(outVertex, vertexMap, edgeMap, stack, nextIndex);
+          av.lowlink = Math.min(av.lowlink, outVertex.lowlink);
+        }
+        else if(outVertex.onstack){
+          // strongly connected component detected, but we will wait till later so that the full cycle can be displayed.
+          // update lowlink in case outputVertex should be considered the root of this component.
+          av.lowlink = Math.min(av.lowlink, outVertex.index);
+        }
+      }
+    }
+
+    if(av.lowlink == av.index ){
+       AnnotatedVertex pop = stack.pop();
+       pop.onstack = false;
+       if(pop != av){
+         // there was something on the stack other than this "av".
+         // this indicates there is a scc/cycle. It comprises all nodes from top of stack to "av"
+         StringBuilder message = new StringBuilder();
+         message.append(av.v.getVertexName() + " <- ");
+         for( ; pop != av; pop = stack.pop()){ 
+           message.append(pop.v.getVertexName() + " <- ");
+           pop.onstack = false;
+         }
+         message.append(av.v.getVertexName());
+         throw new IllegalStateException("DAG contains a cycle: " + message);
+       }
+    }
+  }
+ 
+  
   // create protobuf message describing DAG
   public DAGPlan createDag(){
+    
+    verify(true);
+    
     DAGPlan.Builder jobBuilder = DAGPlan.newBuilder();  
 
     jobBuilder.setName(this.name);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/645c470e/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java
new file mode 100644
index 0000000..4022604
--- /dev/null
+++ b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
+import org.apache.tez.dag.api.EdgeProperty.SourceType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestDAGVerify {
+  
+  private final String dummyProcessorClassName = TestDAGVerify.class.getName();
+  private final String dummyInputClassName = TestDAGVerify.class.getName();
+  private final String dummyOutputClassName = TestDAGVerify.class.getName();
+  private final int dummyTaskCount = 2;
+  
+  //    v1
+  //    |  
+  //    v2
+  @Test
+  public void testVerify1() {
+    Vertex v1 = new Vertex("v1",dummyProcessorClassName, dummyTaskCount);
+    Vertex v2 = new Vertex("v2","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+    Edge e1 = new Edge(v1, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, dummyInputClassName, dummyOutputClassName));
+    DAG dag = new DAG();
+    dag.addVertex(v1);
+    dag.addVertex(v2);
+    dag.addEdge(e1);
+    dag.verify();
+  }
+  
+  //    v1 <----
+  //      |     ^
+  //       v2   ^
+  //      |  |  ^
+  //    v3    v4
+  @Test
+  public void testCycle1() {
+    IllegalStateException ex=null;
+    Vertex v1 = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+    Vertex v2 = new Vertex("v2","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+    Vertex v3 = new Vertex("v3","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+    Vertex v4 = new Vertex("v4","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+    Edge e1 = new Edge(v1, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+    Edge e2 = new Edge(v2, v3, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+    Edge e3 = new Edge(v2, v4, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+    Edge e4 = new Edge(v4, v1, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+    DAG dag = new DAG();
+    dag.addVertex(v1);
+    dag.addVertex(v2);
+    dag.addVertex(v3);
+    dag.addVertex(v4);
+    dag.addEdge(e1);
+    dag.addEdge(e2);
+    dag.addEdge(e3);
+    dag.addEdge(e4);
+    try{
+      dag.verify();
+    }
+    catch (IllegalStateException e){
+      ex = e;
+    }
+    Assert.assertNotNull(ex);
+    System.out.println(ex.getMessage());
+    Assert.assertTrue(ex.getMessage().startsWith("DAG contains a cycle"));
+  }
+  
+//     v1 
+//      |     
+//    -> v2   
+//    ^  | | 
+//    v3    v4
+  @Test
+  public void testCycle2() {
+    IllegalStateException ex=null;
+    Vertex v1 = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+    Vertex v2 = new Vertex("v2","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+    Vertex v3 = new Vertex("v3","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+    Vertex v4 = new Vertex("v4","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+    Edge e1 = new Edge(v1, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+    Edge e2 = new Edge(v2, v3, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+    Edge e3 = new Edge(v2, v4, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+    Edge e4 = new Edge(v3, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+    DAG dag = new DAG();
+    dag.addVertex(v1);
+    dag.addVertex(v2);
+    dag.addVertex(v3);
+    dag.addVertex(v4);
+    dag.addEdge(e1);
+    dag.addEdge(e2);
+    dag.addEdge(e3);
+    dag.addEdge(e4);
+    try{
+      dag.verify();
+    }
+    catch (IllegalStateException e){
+      ex = e;
+    }
+    Assert.assertNotNull(ex);
+    System.out.println(ex.getMessage());
+    Assert.assertTrue(ex.getMessage().startsWith("DAG contains a cycle"));
+  }
+  
+  @Test
+  public void repeatedVertexName() {
+    IllegalStateException ex=null;
+    Vertex v1 = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+    Vertex v1repeat = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+    DAG dag = new DAG();
+    dag.addVertex(v1);
+    dag.addVertex(v1repeat);
+    try {
+      dag.verify();
+    }
+    catch (IllegalStateException e){
+      ex = e;
+    }
+    Assert.assertNotNull(ex);
+    System.out.println(ex.getMessage());
+    Assert.assertTrue(ex.getMessage().startsWith("DAG contains multiple vertices with name"));
+  }
+  
+  //  v1  v2
+  //   |  |     
+  //    v3   
+  @Test
+  public void BinaryInput() {
+    IllegalStateException ex=null;
+    try {
+      Vertex v1 = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+      Vertex v2 = new Vertex("v2","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+      Vertex v3 = new Vertex("v3","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+      Edge e1 = new Edge(v1, v3, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+      Edge e2 = new Edge(v2, v3, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+      DAG dag = new DAG();
+      dag.addVertex(v1);
+      dag.addVertex(v2);
+      dag.addVertex(v3);
+      dag.addEdge(e1);
+      dag.addEdge(e2);
+      dag.verify();
+    }
+    catch (IllegalStateException e){
+      ex = e;
+    }
+    Assert.assertNotNull(ex);
+    System.out.println(ex.getMessage());
+    Assert.assertTrue(ex.getMessage().startsWith("Vertex has inDegree>1"));
+  }
+  
+  //   v1  
+  //  |  |     
+  //  v2  v3 
+  @Test
+  public void BinaryOutput() {
+    IllegalStateException ex=null;
+    try {
+      Vertex v1 = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+      Vertex v2 = new Vertex("v2","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+      Vertex v3 = new Vertex("v3","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+      Edge e1 = new Edge(v1, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+      Edge e2 = new Edge(v1, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+      DAG dag = new DAG();
+      dag.addVertex(v1);
+      dag.addVertex(v2);
+      dag.addVertex(v3);
+      dag.addEdge(e1);
+      dag.addEdge(e2);
+      dag.verify();
+    }
+    catch (IllegalStateException e){
+      ex = e;
+    }
+    Assert.assertNotNull(ex);
+    System.out.println(ex.getMessage());
+    Assert.assertTrue(ex.getMessage().startsWith("Vertex has outDegree>1"));
+  }
+}


[45/50] [abbrv] git commit: TEZ-178. Change YARN object construction to be based on newInstance in various classes instead of BuilderUtils. (sseth)

Posted by ss...@apache.org.
TEZ-178. Change YARN object construction to be based on newInstance in
various classes instead of BuilderUtils. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/9f040cf8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/9f040cf8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/9f040cf8

Branch: refs/heads/master
Commit: 9f040cf8b33600233be2bd2b9795406816c687e4
Parents: 8652329
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jun 3 13:35:56 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jun 3 13:35:56 2013 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/tez/common/IDUtils.java   |    5 +-
 .../java/org/apache/tez/dag/records/TezDAGID.java  |    5 +-
 .../org/apache/tez/records/TezContainerId.java     |   11 ++-
 .../org/apache/tez/dag/api/DagTypeConverters.java  |    5 +-
 .../org/apache/tez/dag/api/oldrecords/AMInfo.java  |    2 +-
 .../org/apache/tez/dag/app/MRRExampleHelper.java   |   10 +-
 .../apache/tez/dag/app/TaskHeartbeatHandler.java   |    2 +-
 .../tez/dag/app/dag/TaskAttemptStateInternal.java  |    2 +-
 .../TaskAttemptEventContainerTerminating.java      |    2 +-
 .../app/dag/event/TaskAttemptEventFailRequest.java |    2 +-
 .../app/dag/event/TaskAttemptEventNodeFailed.java  |    2 +-
 .../tez/dag/app/dag/impl/DAGSchedulerMRR.java      |    8 +-
 .../dag/app/dag/impl/DAGSchedulerNaturalOrder.java |    8 +-
 .../tez/dag/app/rm/AMSchedulerEventTAEnded.java    |    2 +-
 .../app/rm/AMSchedulerEventTALaunchRequest.java    |    2 +-
 .../rm/container/AMContainerEventNodeFailed.java   |    2 +-
 .../dag/app/rm/container/AMContainerHelpers.java   |    3 +-
 .../tez/dag/app/rm/container/AMContainerMap.java   |    2 +-
 .../app/rm/node/AMNodeEventTaskAttemptEnded.java   |    2 +-
 .../tez/dag/app/rm/node/AMNodeEventType.java       |    2 +-
 .../org/apache/tez/dag/app/rm/node/AMNodeMap.java  |    2 +-
 .../org/apache/tez/dag/utils/TezBuilderUtils.java  |    2 +-
 .../apache/tez/dag/app/dag/impl/TestDAGImpl.java   |    6 +-
 .../tez/dag/app/dag/impl/TestTaskAttempt.java      |   74 +++++++-------
 .../apache/tez/dag/app/dag/impl/TestTaskImpl.java  |    5 +-
 .../tez/dag/app/dag/impl/TestVertexImpl.java       |    6 +-
 .../apache/tez/dag/app/rm/TestTaskScheduler.java   |    6 +-
 .../tez/dag/app/rm/container/TestAMContainer.java  |   15 ++--
 .../apache/tez/mapreduce/hadoop/IDConverter.java   |    3 +-
 .../hadoop/MultiStageMRConfToTezTranslator.java    |    2 +-
 .../org/apache/tez/mapreduce/TezTestUtils.java     |   10 +-
 .../org/apache/tez/mapreduce/NotRunningJob.java    |    1 -
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |    7 +-
 33 files changed, 107 insertions(+), 111 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/IDUtils.java b/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
index e2cbf35..e94d939 100644
--- a/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
+++ b/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
@@ -19,7 +19,6 @@
 package org.apache.tez.common;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezID;
 import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -41,7 +40,7 @@ public class IDUtils {
       String[] parts = str.split("_");
       if(parts.length == 6) {
         if(parts[0].equals(TezTaskID.TASK)) {
-          ApplicationId appId = BuilderUtils.newApplicationId(
+          ApplicationId appId = ApplicationId.newInstance(
               Long.valueOf(parts[1]), Integer.parseInt(parts[2]));
           TezDAGID dagId = new TezDAGID(appId, Integer.parseInt(parts[3]));
           TezVertexID vId = new TezVertexID(dagId, Integer.parseInt(parts[4]));
@@ -71,7 +70,7 @@ public class IDUtils {
       String[] parts = str.split(Character.toString(TezID.SEPARATOR));
       if(parts.length == 7) {
         if(parts[0].equals(TezTaskAttemptID.ATTEMPT)) {
-          ApplicationId appId = BuilderUtils.newApplicationId(
+          ApplicationId appId = ApplicationId.newInstance(
               Long.valueOf(parts[1]), Integer.parseInt(parts[2]));
           TezDAGID dagId = new TezDAGID(appId, Integer.parseInt(parts[3]));
           TezVertexID vId = new TezVertexID(dagId, Integer.parseInt(parts[4]));

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
index 733a908..9a0a409 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
@@ -24,7 +24,6 @@ import java.io.IOException;
 import java.text.NumberFormat;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 
 
 /**
@@ -71,7 +70,7 @@ public class TezDAGID extends TezID {
    * @param id the dag number
    */
   public TezDAGID(String yarnRMIdentifier, int appId, int id) {
-    this(BuilderUtils.newApplicationId(Long.valueOf(yarnRMIdentifier),
+    this(ApplicationId.newInstance(Long.valueOf(yarnRMIdentifier),
         appId), id);
   }
 
@@ -117,7 +116,7 @@ public class TezDAGID extends TezID {
 
   @Override
   public void readFields(DataInput in) throws IOException {
-    applicationId = BuilderUtils.newApplicationId(in.readLong(), in.readInt());
+    applicationId = ApplicationId.newInstance(in.readLong(), in.readInt());
     super.readFields(in);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java b/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
index 24c1e8a..193b979 100644
--- a/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
+++ b/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
@@ -24,8 +24,9 @@ import java.io.IOException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 
 //TODO EVENTUALLY Once everything is on PB, get rid of this.
 //Alternately have the PB interfaces implement Writable.
@@ -58,8 +59,12 @@ public class TezContainerId implements Writable {
     int appId = in.readInt();
     int appAttemptId = in.readInt();
     int id = in.readInt();
-    this.containerId = BuilderUtils.newContainerId(appId, appAttemptId,
-        timestamp, id);
+
+    ApplicationId applicationId = ApplicationId.newInstance(timestamp, appId);
+    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId
+        .newInstance(applicationId, appAttemptId);
+
+    this.containerId = ContainerId.newInstance(applicationAttemptId, id);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index ba42e23..6437080 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
 import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeConnectionPattern;
@@ -207,7 +206,7 @@ public class DagTypeConverters {
 
   public static Resource CreateResourceRequestFromTaskConfig(
       PlanTaskConfiguration taskConfig) {
-    return BuilderUtils.newResource(taskConfig.getMemoryMb(), taskConfig.getVirtualCores());
+    return Resource.newInstance(taskConfig.getMemoryMb(), taskConfig.getVirtualCores());
   }
 
   public static Map<String, String> createSettingsMapFromDAGPlan(
@@ -218,4 +217,4 @@ public class DagTypeConverters {
     }
     return map;
   }   
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
index 7726d7b..6cb3df0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
@@ -35,4 +35,4 @@ public interface AMInfo {
   public void setNodeManagerHost(String nmHost);
   public void setNodeManagerPort(int nmPort);
   public void setNodeManagerHttpPort(int mnHttpPort);
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
index 92f0e87..35ae558 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
@@ -16,7 +16,6 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.Edge;
 import org.apache.tez.dag.api.EdgeProperty;
@@ -126,12 +125,13 @@ public class MRRExampleHelper {
    reduce2Rsrcs.put(MRJobConfig.JOB_CONF_FILE, jobRsrcs.get(MRJobConfig.JOB_CONF_FILE));
    reduce2Rsrcs.put(getConfFileName("reduce2"), jobRsrcs.get(getConfFileName("reduce2")));
 
-   Resource mapResource = BuilderUtils.newResource(
+    Resource mapResource = Resource.newInstance(
         MRJobConfig.DEFAULT_MAP_MEMORY_MB,
         MRJobConfig.DEFAULT_MAP_CPU_VCORES);
+   
    mapVertex.setTaskResource(mapResource);
    mapVertex.setTaskLocalResources(mapRsrcs);
-   Resource reduceResource = BuilderUtils.newResource(
+   Resource reduceResource = Resource.newInstance(
        MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
        MRJobConfig.DEFAULT_REDUCE_CPU_VCORES);
    reduce1Vertex.setTaskResource(reduceResource);
@@ -182,12 +182,12 @@ public class MRRExampleHelper {
    reduceRsrcs.put(MRJobConfig.JOB_CONF_FILE, jobRsrcs.get(MRJobConfig.JOB_CONF_FILE));
    reduceRsrcs.put(getConfFileName("reduce"), jobRsrcs.get(MRJobConfig.JOB_CONF_FILE));
 
-   Resource mapResource = BuilderUtils.newResource(
+   Resource mapResource = Resource.newInstance(
         MRJobConfig.DEFAULT_MAP_MEMORY_MB,
         MRJobConfig.DEFAULT_MAP_CPU_VCORES);
    mapVertex.setTaskResource(mapResource);
    mapVertex.setTaskLocalResources(mapRsrcs);
-   Resource reduceResource = BuilderUtils.newResource(
+   Resource reduceResource = Resource.newInstance(
        MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
        MRJobConfig.DEFAULT_REDUCE_CPU_VCORES);
    reduceVertex.setTaskResource(reduceResource);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
index 7239484..b6fdc53 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
@@ -65,4 +65,4 @@ public class TaskHeartbeatHandler extends HeartbeatHandlerBase<TezTaskAttemptID>
     eventHandler.handle(new TaskAttemptEvent(attemptId,
         TaskAttemptEventType.TA_TIMED_OUT));
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
index 3633b08..9ad5460 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
@@ -36,4 +36,4 @@ public enum TaskAttemptStateInternal {
   KILLED,
   FAILED,
   SUCCEEDED
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
index 8c88cb9..7da6e14 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
@@ -35,4 +35,4 @@ public class TaskAttemptEventContainerTerminating extends TaskAttemptEvent
     return this.message;
   }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
index bba8027..b2f9cfb 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
@@ -32,4 +32,4 @@ public class TaskAttemptEventFailRequest extends TaskAttemptEvent {
   public String getMessage() {
     return this.message;
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
index 4fc718d..ee143bb 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
@@ -33,4 +33,4 @@ public class TaskAttemptEventNodeFailed extends TaskAttemptEvent {
     return this.message;
   }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
index d404155..4f51c02 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
@@ -20,8 +20,8 @@ package org.apache.tez.dag.app.dag.impl;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.DAGScheduler;
@@ -124,9 +124,9 @@ public class DAGSchedulerMRR implements DAGScheduler {
              " with depth " + vertexDistanceFromRoot + 
              " at priority " + priority);
 
-    TaskAttemptEventSchedule attemptEvent = 
-        new TaskAttemptEventSchedule(attempt.getID(), 
-                                      BuilderUtils.newPriority(priority));
+    TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule(
+        attempt.getID(), Priority.newInstance(priority)); 
+                                      
     sendEvent(attemptEvent);
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
index 9e8729e..6fec907 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
@@ -20,8 +20,8 @@ package org.apache.tez.dag.app.dag.impl;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.DAGScheduler;
 import org.apache.tez.dag.app.dag.TaskAttempt;
@@ -66,9 +66,9 @@ public class DAGSchedulerNaturalOrder implements DAGScheduler {
 
     LOG.info("Scheduling " + attempt.getID() + " at priority " + priority);
     
-    TaskAttemptEventSchedule attemptEvent = 
-        new TaskAttemptEventSchedule(attempt.getID(), 
-                                      BuilderUtils.newPriority(priority));
+    TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule(
+        attempt.getID(), Priority.newInstance(priority));
+                                      
     sendEvent(attemptEvent);
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
index b2a90f2..90e76b7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
@@ -51,4 +51,4 @@ public class AMSchedulerEventTAEnded extends AMSchedulerEvent {
   public ContainerId getUsedContainerId() {
     return this.containerId;
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
index b595c2b..313ee7d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
@@ -130,4 +130,4 @@ public class AMSchedulerEventTALaunchRequest extends AMSchedulerEvent {
    * - Credentials, tokens etc are identical.
    * + Command - dependent on map / reduce java.opts
    */
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java
index c1c603c..9f5d39c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java
@@ -36,4 +36,4 @@ public class AMContainerEventNodeFailed extends AMContainerEvent implements
     return message;
   }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index c17f7f2..6156e33 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.app.AppContext;
@@ -76,7 +75,7 @@ public class AMContainerHelpers {
     long resourceSize = fstat.getLen();
     long resourceModificationTime = fstat.getModificationTime();
 
-    return BuilderUtils.newLocalResource(resourceURL, type, visibility,
+    return LocalResource.newInstance(resourceURL, type, visibility,
         resourceSize, resourceModificationTime);
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
index 708838c..5f5a7be 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
@@ -72,4 +72,4 @@ public class AMContainerMap extends AbstractService implements
   public Collection<AMContainer> values() {
     return containerMap.values();
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
index cc2b7f7..c823236 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
@@ -50,4 +50,4 @@ public class AMNodeEventTaskAttemptEnded extends AMNodeEvent {
   public TezTaskAttemptID getTaskAttemptId() {
     return this.taskAttemptId;
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java
index 60534fd..421e482 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java
@@ -38,4 +38,4 @@ public enum AMNodeEventType {
   // Producer: AMNode - Will not reach AMNodeImpl. Used to compute whether
   // blacklisting should be ignored.
   N_NODE_WAS_BLACKLISTED
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
index 206244e..a9256aa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
@@ -210,4 +210,4 @@ public class AMNodeMap extends AbstractService implements
   public boolean isBlacklistingIgnored() {
     return this.ignoreBlacklisting;
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
index 60ec0a0..a734bb8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
@@ -51,4 +51,4 @@ public class TezBuilderUtils {
     return new TezTaskID(newVertexID(dagId, vertexId), taskId);
   }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index 9e94cbd..9588099 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -27,9 +27,9 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.SystemClock;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.event.DrainDispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
@@ -343,8 +343,8 @@ public class TestDAGImpl {
   @Before
   public void setup() {
     conf = new TezConfiguration();
-    appAttemptId = BuilderUtils.newApplicationAttemptId(
-        BuilderUtils.newApplicationId(100, 1), 1);
+    appAttemptId = ApplicationAttemptId.newInstance(
+        ApplicationId.newInstance(100, 1), 1);
     dagId = new TezDAGID(appAttemptId.getApplicationId(), 1);
     Assert.assertNotNull(dagId);
     dagPlan = createTestDAGPlan();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index 9601418..516c656 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -49,10 +49,10 @@ import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.common.TezTaskContext;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
@@ -131,7 +131,7 @@ public class TestTaskAttempt {
         mock(TaskAttemptListener.class), 1, new TezConfiguration(),
         mock(Token.class), new Credentials(), new SystemClock(),
         mock(TaskHeartbeatHandler.class), mock(AppContext.class),
-        MAP_PROCESSOR_NAME, locationHint, BuilderUtils.newResource(1024, 1),
+        MAP_PROCESSOR_NAME, locationHint, Resource.newInstance(1024, 1),
         new HashMap<String, LocalResource>(), new HashMap<String, String>(),
         "", false);
 
@@ -175,7 +175,7 @@ public class TestTaskAttempt {
         mock(TaskAttemptListener.class), 1, new TezConfiguration(),
         mock(Token.class), new Credentials(), new SystemClock(),
         mock(TaskHeartbeatHandler.class), mock(AppContext.class),
-        MAP_PROCESSOR_NAME, locationHint, BuilderUtils.newResource(1024, 1),
+        MAP_PROCESSOR_NAME, locationHint, Resource.newInstance(1024, 1),
         new HashMap<String, LocalResource>(), new HashMap<String, String>(),
         "", false);
     TaskAttemptImpl spyTa = spy(taImpl);
@@ -222,8 +222,8 @@ public class TestTaskAttempt {
   // Configuration conf = new Configuration();
   // conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
   // conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
-  // app.setClusterInfo(new ClusterInfo(BuilderUtils
-  // .newResource(minContainerSize, 1), BuilderUtils.newResource(10240,1)));
+  // app.setClusterInfo(new ClusterInfo(Resource.newInstance(minContainerSize, 1),
+  // Resource.newInstance(10240,1)));
   //
   // Job job = app.submit(conf);
   // app.waitForState(job, JobState.RUNNING);
@@ -294,8 +294,8 @@ public class TestTaskAttempt {
   // Ensure the dag does not go into an error state if a attempt kill is
   // received while STARTING
   public void testLaunchFailedWhileKilling() throws Exception {
-    ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
-    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+    ApplicationId appId = ApplicationId.newInstance(1, 2);
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
         appId, 0);
     TezDAGID dagID = new TezDAGID(appId, 1);
     TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -313,7 +313,7 @@ public class TestTaskAttempt {
 
     TaskLocationHint locationHint = new TaskLocationHint(
         new String[] { "127.0.0.1" }, null);
-    Resource resource = BuilderUtils.newResource(1024, 1);
+    Resource resource = Resource.newInstance(1024, 1);
     Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
     Map<String, String> environment = new HashMap<String, String>();
     String javaOpts = "";
@@ -327,14 +327,14 @@ public class TestTaskAttempt {
         MAP_PROCESSOR_NAME, locationHint, resource, localResources,
         environment, javaOpts, false);
 
-    NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
-    ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+    ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
     Container container = mock(Container.class);
     when(container.getId()).thenReturn(contId);
     when(container.getNodeId()).thenReturn(nid);
 
-    taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, BuilderUtils
-        .newPriority(3)));
+    taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, Priority
+        .newInstance(3)));
     // At state STARTING.
     taImpl.handle(new TaskAttemptEventKillRequest(taskAttemptID, null));
     // At some KILLING state.
@@ -348,8 +348,8 @@ public class TestTaskAttempt {
   // Ensure ContainerTerminating and ContainerTerminated is handled correctly by
   // the TaskAttempt
   public void testContainerTerminationWhileRunning() throws Exception {
-    ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
-    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+    ApplicationId appId = ApplicationId.newInstance(1, 2);
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
         appId, 0);
     TezDAGID dagID = new TezDAGID(appId, 1);
     TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -367,13 +367,13 @@ public class TestTaskAttempt {
 
     TaskLocationHint locationHint = new TaskLocationHint(
         new String[] { "127.0.0.1" }, null);
-    Resource resource = BuilderUtils.newResource(1024, 1);
+    Resource resource = Resource.newInstance(1024, 1);
     Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
     Map<String, String> environment = new HashMap<String, String>();
     String javaOpts = "";
 
-    NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
-    ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+    ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
     Container container = mock(Container.class);
     when(container.getId()).thenReturn(contId);
     when(container.getNodeId()).thenReturn(nid);
@@ -443,8 +443,8 @@ public class TestTaskAttempt {
   @Test
   // Ensure ContainerTerminated is handled correctly by the TaskAttempt
   public void testContainerTerminatedWhileRunning() throws Exception {
-    ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
-    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+    ApplicationId appId = ApplicationId.newInstance(1, 2);
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
         appId, 0);
     TezDAGID dagID = new TezDAGID(appId, 1);
     TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -462,13 +462,13 @@ public class TestTaskAttempt {
 
     TaskLocationHint locationHint = new TaskLocationHint(
         new String[] { "127.0.0.1" }, null);
-    Resource resource = BuilderUtils.newResource(1024, 1);
+    Resource resource = Resource.newInstance(1024, 1);
     Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
     Map<String, String> environment = new HashMap<String, String>();
     String javaOpts = "";
 
-    NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
-    ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+    ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
     Container container = mock(Container.class);
     when(container.getId()).thenReturn(contId);
     when(container.getNodeId()).thenReturn(nid);
@@ -508,8 +508,8 @@ public class TestTaskAttempt {
   @Test
   // Ensure ContainerTerminated is handled correctly by the TaskAttempt
   public void testContainerTerminatedWhileCommitting() throws Exception {
-    ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
-    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+    ApplicationId appId = ApplicationId.newInstance(1, 2);
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
         appId, 0);
     TezDAGID dagID = new TezDAGID(appId, 1);
     TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -527,13 +527,13 @@ public class TestTaskAttempt {
 
     TaskLocationHint locationHint = new TaskLocationHint(
         new String[] { "127.0.0.1" }, null);
-    Resource resource = BuilderUtils.newResource(1024, 1);
+    Resource resource = Resource.newInstance(1024, 1);
     Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
     Map<String, String> environment = new HashMap<String, String>();
     String javaOpts = "";
 
-    NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
-    ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+    ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
     Container container = mock(Container.class);
     when(container.getId()).thenReturn(contId);
     when(container.getNodeId()).thenReturn(nid);
@@ -575,8 +575,8 @@ public class TestTaskAttempt {
   // Ensure ContainerTerminating and ContainerTerminated is handled correctly by
   // the TaskAttempt
   public void testContainerTerminatedAfterSuccess() throws Exception {
-    ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
-    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+    ApplicationId appId = ApplicationId.newInstance(1, 2);
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
         appId, 0);
     TezDAGID dagID = new TezDAGID(appId, 1);
     TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -594,13 +594,13 @@ public class TestTaskAttempt {
 
     TaskLocationHint locationHint = new TaskLocationHint(
         new String[] { "127.0.0.1" }, null);
-    Resource resource = BuilderUtils.newResource(1024, 1);
+    Resource resource = Resource.newInstance(1024, 1);
     Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
     Map<String, String> environment = new HashMap<String, String>();
     String javaOpts = "";
 
-    NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
-    ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+    ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
     Container container = mock(Container.class);
     when(container.getId()).thenReturn(contId);
     when(container.getNodeId()).thenReturn(nid);
@@ -667,8 +667,8 @@ public class TestTaskAttempt {
   // Verifies that multiple TooManyFetchFailures are handled correctly by the
   // TaskAttempt.
   public void testMultipleTooManyFetchFailures() throws Exception {
-    ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
-    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+    ApplicationId appId = ApplicationId.newInstance(1, 2);
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
         appId, 0);
     TezDAGID dagID = new TezDAGID(appId, 1);
     TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -687,13 +687,13 @@ public class TestTaskAttempt {
 
     TaskLocationHint locationHint = new TaskLocationHint(
         new String[] { "127.0.0.1" }, null);
-    Resource resource = BuilderUtils.newResource(1024, 1);
+    Resource resource = Resource.newInstance(1024, 1);
     Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
     Map<String, String> environment = new HashMap<String, String>();
     String javaOpts = "";
 
-    NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
-    ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+    ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
     Container container = mock(Container.class);
     when(container.getId()).thenReturn(contId);
     when(container.getNodeId()).thenReturn(nid);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 44e7686..f2ab880 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.event.InlineDispatcher;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
 import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
@@ -101,11 +100,11 @@ public class TestTaskImpl {
     clock = new SystemClock();
     locationHint = new TaskLocationHint(new String[1], new String[1]);
 
-    appId = BuilderUtils.newApplicationId(System.currentTimeMillis(), 1);
+    appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
     dagId = new TezDAGID(appId, 1);
     vertexId = new TezVertexID(dagId, 1);
     appContext = mock(AppContext.class);
-    taskResource = BuilderUtils.newResource(1024, 1);
+    taskResource = Resource.newInstance(1024, 1);
     localResources = new HashMap<String, LocalResource>();
     environment = new HashMap<String, String>();
     javaOpts = "";

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 41b500f..e3af5b3 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -34,9 +34,9 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.SystemClock;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.event.DrainDispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
@@ -463,8 +463,8 @@ public class TestVertexImpl {
 
   @Before
   public void setup() {
-    appAttemptId = BuilderUtils.newApplicationAttemptId(
-        BuilderUtils.newApplicationId(100, 1), 1);
+    appAttemptId = ApplicationAttemptId.newInstance(
+        ApplicationId.newInstance(100, 1), 1);
     dagId = new TezDAGID(appAttemptId.getApplicationId(), 1);
     dagPlan = createTestDAGPlan();
     dispatcher = new DrainDispatcher();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
index 8d52e9a..7036e09 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -40,7 +41,6 @@ import org.apache.hadoop.yarn.client.AMRMClientAsync;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.RackResolver;
 import org.apache.tez.dag.app.rm.TaskScheduler.CookieContainerRequest;
 import org.apache.tez.dag.app.rm.TaskScheduler.TaskSchedulerAppCallback;
@@ -68,8 +68,8 @@ public class TestTaskScheduler {
                                                   mock(AMRMClientAsync.class);
     
     ApplicationAttemptId attemptId = 
-        BuilderUtils.newApplicationAttemptId(
-                                  BuilderUtils.newApplicationId(1234, 0), 0);
+        ApplicationAttemptId.newInstance(
+                                  ApplicationId.newInstance(1234, 0), 0);
     String appHost = "host";
     int appPort = 0;
     String appUrl = "url";

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index cf01f43..332f737 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.common.TezTaskContext;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.app.AppContext;
@@ -831,14 +830,14 @@ public class TestAMContainer {
     
     @SuppressWarnings("unchecked")
     public WrappedContainer() {
-      applicationID = BuilderUtils.newApplicationId(rmIdentifier, 1);
-      appAttemptID = BuilderUtils.newApplicationAttemptId(applicationID, 1);
-      containerID = BuilderUtils.newContainerId(appAttemptID, 1);
-      nodeID = BuilderUtils.newNodeId("host", 12500);
+      applicationID = ApplicationId.newInstance(rmIdentifier, 1);
+      appAttemptID = ApplicationAttemptId.newInstance(applicationID, 1);
+      containerID = ContainerId.newInstance(appAttemptID, 1);
+      nodeID = NodeId.newInstance("host", 12500);
       nodeHttpAddress = "host:12501";
-      resource = BuilderUtils.newResource(1024, 1);
-      priority = BuilderUtils.newPriority(1);
-      container = BuilderUtils.newContainer(containerID, nodeID,
+      resource = Resource.newInstance(1024, 1);
+      priority = Priority.newInstance(1);
+      container = Container.newInstance(containerID, nodeID,
           nodeHttpAddress, resource, priority, null);
 
       chh = mock(ContainerHeartbeatHandler.class);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
index 4046596..f7c4f50 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.mapred.TaskAttemptID;
 import org.apache.hadoop.mapred.TaskID;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
@@ -55,7 +54,7 @@ public class IDConverter {
   // FIXME hardcoded assumption that one app is one dag
   public static TezDAGID fromMRJobId(
       org.apache.hadoop.mapreduce.JobID jobId) {
-    return new TezDAGID(BuilderUtils.newApplicationId(
+    return new TezDAGID(ApplicationId.newInstance(
         Long.valueOf(jobId.getJtIdentifier()), jobId.getId()), 1);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
index c444314..9f1ae88 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
@@ -189,4 +189,4 @@ public class MultiStageMRConfToTezTranslator {
       }
     }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
index 2e823e6..9a3a666 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
@@ -17,7 +17,7 @@
  */
 package org.apache.tez.mapreduce;
 
-import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
@@ -31,7 +31,7 @@ public class TezTestUtils {
         new TezTaskID(
             new TezVertexID(
                 new TezDAGID(
-                    BuilderUtils.newApplicationId(0, jobId), jobId),
+                    ApplicationId.newInstance(0, jobId), jobId),
                     vertexId),
                     taskId)
         , taskAttemptId);
@@ -41,20 +41,20 @@ public class TezTestUtils {
       int vertexId, int taskId) {
     return new TezTaskID(
         new TezVertexID(new TezDAGID(
-            BuilderUtils.newApplicationId(0, jobId),
+            ApplicationId.newInstance(0, jobId),
             jobId), vertexId),
             taskId);
   }
   
   public static TezDAGID getMockJobId(int jobId) {
     return new TezDAGID(
-        BuilderUtils.newApplicationId(0, jobId), jobId);
+        ApplicationId.newInstance(0, jobId), jobId);
   }
   
   public static TezVertexID getMockVertexId(int jobId, int vId) {
     return new TezVertexID(
         new TezDAGID(
-            BuilderUtils.newApplicationId(0, jobId), jobId),
+            ApplicationId.newInstance(0, jobId), jobId),
             vId);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
index ba7ab5a..c29102b 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
@@ -68,7 +68,6 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 
 public class NotRunningJob implements MRClientProtocol {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 572437b..e2ef5b6 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -89,7 +89,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.tez.dag.api.DAG;
@@ -486,7 +485,7 @@ public class YARNRunner implements ClientProtocol {
     Map<String, String> reduceEnv = new HashMap<String, String>();
     setupMapReduceEnv(conf, reduceEnv, false);
 
-    Resource reduceResource = BuilderUtils.newResource(conf.getInt(
+    Resource reduceResource = Resource.newInstance(conf.getInt(
         MRJobConfig.REDUCE_MEMORY_MB, MRJobConfig.DEFAULT_REDUCE_MEMORY_MB),
         conf.getInt(MRJobConfig.REDUCE_CPU_VCORES,
             MRJobConfig.DEFAULT_REDUCE_CPU_VCORES));
@@ -559,7 +558,7 @@ public class YARNRunner implements ClientProtocol {
     TaskLocationHint[] inputSplitLocations =
         getMapLocationHintsFromInputSplits(jobId, fs, jobConf, jobSubmitDir);
 
-    Resource mapResource = BuilderUtils.newResource(
+    Resource mapResource = Resource.newInstance(
         jobConf.getInt(MRJobConfig.MAP_MEMORY_MB,
             MRJobConfig.DEFAULT_MAP_MEMORY_MB),
         jobConf.getInt(MRJobConfig.MAP_CPU_VCORES,
@@ -602,7 +601,7 @@ public class YARNRunner implements ClientProtocol {
       Map<String, String> reduceEnv = new HashMap<String, String>();
       setupMapReduceEnv(jobConf, reduceEnv, false);
 
-      Resource reduceResource = BuilderUtils.newResource(
+      Resource reduceResource = Resource.newInstance(
           jobConf.getInt(MRJobConfig.REDUCE_MEMORY_MB,
               MRJobConfig.DEFAULT_REDUCE_MEMORY_MB),
           jobConf.getInt(MRJobConfig.REDUCE_CPU_VCORES,


[41/50] [abbrv] git commit: TEZ-172. TestVertexImpl hangs (intermittent failure).

Posted by ss...@apache.org.
TEZ-172. TestVertexImpl hangs (intermittent failure).


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/d94d37af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/d94d37af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/d94d37af

Branch: refs/heads/master
Commit: d94d37afbe9a0cc045e2c8717c93fa5446b589b9
Parents: d3b1921
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri May 31 17:10:14 2013 -0700
Committer: Mike Liddell <ml...@apache.org>
Committed: Fri May 31 18:09:37 2013 -0700

----------------------------------------------------------------------
 pom.xml                                            |    3 +-
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |    6 +-
 .../apache/tez/dag/app/dag/impl/TestDAGImpl.java   |   68 +++--
 .../tez/dag/app/dag/impl/TestVertexImpl.java       |  262 ++++++++++-----
 4 files changed, 218 insertions(+), 121 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d94d37af/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 721df2f..4c5d0c5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -325,7 +325,8 @@
           <artifactId>maven-surefire-plugin</artifactId>
           <version>2.14.1</version>
           <configuration>
-            <forkMode>always</forkMode>
+            <forkCount>1</forkCount>
+            <reuseForks>false</reuseForks>
             <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
             <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
             <redirectTestOutputToFile>true</redirectTestOutputToFile>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d94d37af/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index c7bceb6..b06e264 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -613,7 +613,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
          getStateMachine().doTransition(event.getType(), event);
       } catch (InvalidStateTransitonException e) {
         String message = "Invalid event " + event.getType() +
-            " on vertex " + this.vertexId +
+            " on vertex " + this.vertexName +
+            " with vertexId " + this.vertexId +
             " at current state " + oldState;
         LOG.error("Can't handle " + message, e);
         addDiagnostic(message);
@@ -630,6 +631,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     finally {
       writeLock.unlock();
     }
+    LOG.info("DEBUG: Finished processing VertexEvent " + event.getVertexId()
+        + " of type " + event.getType() + " while in state "
+        + getInternalState() + ". Event: " + event);
   }
 
   private VertexState getInternalState() {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d94d37af/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index 3adcd8b..9e94cbd 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -373,19 +373,23 @@ public class TestDAGImpl {
 
   @After
   public void teardown() {
-    dagPlan = null;
-    dag = null;
     dispatcher.await();
     dispatcher.stop();
+    dagPlan = null;
+    dag = null;
   }
 
   private void initDAG(DAGImpl dag) {
-    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_INIT));
+    dag.handle(
+        new DAGEvent(dagId, DAGEventType.DAG_INIT));
     Assert.assertEquals(DAGState.INITED, dag.getState());
   }
 
+  @SuppressWarnings("unchecked")
   private void startDAG(DAGImpl dag) {
-    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_START));
+    dispatcher.getEventHandler().handle(
+        new DAGEvent(dagId, DAGEventType.DAG_START));
+    dispatcher.await();
     Assert.assertEquals(DAGState.RUNNING, dag.getState());
   }
 
@@ -432,9 +436,9 @@ public class TestDAGImpl {
 
     TezVertexID vId = new TezVertexID(dagId, 1);
     Vertex v = dag.getVertex(vId);
-    ((EventHandler<VertexEvent>) v).handle(new VertexEventTaskCompleted(
+    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(
         new TezTaskID(vId, 0), TaskState.SUCCEEDED));
-    ((EventHandler<VertexEvent>) v).handle(new VertexEventTaskCompleted(
+    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(
         new TezTaskID(vId, 1), TaskState.SUCCEEDED));
     dispatcher.await();
 
@@ -442,12 +446,14 @@ public class TestDAGImpl {
     Assert.assertEquals(1, dag.getSuccessfulVertices());
   }
 
+  @SuppressWarnings("unchecked")
   public void testKillStartedDAG() {
     initDAG(dag);
     startDAG(dag);
     dispatcher.await();
 
-    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+    dispatcher.getEventHandler().handle(
+        new DAGEvent(dagId, DAGEventType.DAG_KILL));
     dispatcher.await();
 
     Assert.assertEquals(DAGState.KILLED, dag.getState());
@@ -479,7 +485,7 @@ public class TestDAGImpl {
     Assert.assertEquals(VertexState.SUCCEEDED, v0.getState());
     Assert.assertEquals(VertexState.RUNNING, v1.getState());
 
-    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+    dispatcher.getEventHandler().handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
     dispatcher.await();
 
     Assert.assertEquals(DAGState.KILL_WAIT, dag.getState());
@@ -493,13 +499,16 @@ public class TestDAGImpl {
     Assert.assertEquals(1, dag.getSuccessfulVertices());
   }
 
+  @SuppressWarnings("unchecked")
   @Test
   public void testInvalidEvent() {
-    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_START));
+    dispatcher.getEventHandler().handle(
+        new DAGEvent(dagId, DAGEventType.DAG_START));
     dispatcher.await();
     Assert.assertEquals(DAGState.ERROR, dag.getState());
   }
 
+  @SuppressWarnings("unchecked")
   @Test
   @Ignore
   public void testVertexSuccessfulCompletionUpdates() {
@@ -508,28 +517,29 @@ public class TestDAGImpl {
     dispatcher.await();
 
     for (int i = 0; i < 6; ++i) {
-      dag.handle(new DAGEventVertexCompleted(
+      dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
           new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
     }
     dispatcher.await();
     Assert.assertEquals(DAGState.RUNNING, dag.getState());
     Assert.assertEquals(1, dag.getSuccessfulVertices());
 
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 2), VertexState.SUCCEEDED));
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 3), VertexState.SUCCEEDED));
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 4), VertexState.SUCCEEDED));
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 5), VertexState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(DAGState.SUCCEEDED, dag.getState());
     Assert.assertEquals(6, dag.getSuccessfulVertices());
   }
 
+  @SuppressWarnings("unchecked")
   @Test
   @Ignore
   public void testVertexFailureHandling() {
@@ -537,14 +547,14 @@ public class TestDAGImpl {
     startDAG(dag);
     dispatcher.await();
 
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(DAGState.RUNNING, dag.getState());
 
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 2), VertexState.FAILED));
     dispatcher.await();
     Assert.assertEquals(DAGState.FAILED, dag.getState());
@@ -558,6 +568,7 @@ public class TestDAGImpl {
     }
   }
 
+  @SuppressWarnings("unchecked")
   @Test
   @Ignore
   public void testDAGKill() {
@@ -565,17 +576,18 @@ public class TestDAGImpl {
     startDAG(dag);
     dispatcher.await();
 
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(DAGState.RUNNING, dag.getState());
 
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
-    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+    dispatcher.getEventHandler().handle(
+        new DAGEvent(dagId, DAGEventType.DAG_KILL));
 
     for (int i = 2; i < 6; ++i) {
-      dag.handle(new DAGEventVertexCompleted(
+      dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
           new TezVertexID(dagId, i), VertexState.SUCCEEDED));
     }
     dispatcher.await();
@@ -584,29 +596,31 @@ public class TestDAGImpl {
     Assert.assertEquals(1, dagFinishEventHandler.dagFinishEvents);
   }
 
+  @SuppressWarnings("unchecked")
   @Test
   public void testDAGKillPending() {
     initDAG(dag);
     startDAG(dag);
     dispatcher.await();
 
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(DAGState.RUNNING, dag.getState());
 
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
-    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+    dispatcher.getEventHandler().handle(
+        new DAGEvent(dagId, DAGEventType.DAG_KILL));
 
     for (int i = 2; i < 5; ++i) {
-      dag.handle(new DAGEventVertexCompleted(
+      dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
           new TezVertexID(dagId, i), VertexState.SUCCEEDED));
     }
     dispatcher.await();
     Assert.assertEquals(DAGState.KILL_WAIT, dag.getState());
 
-    dag.handle(new DAGEventVertexCompleted(
+    dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
         new TezVertexID(dagId, 5), VertexState.KILLED));
     dispatcher.await();
     Assert.assertEquals(DAGState.KILLED, dag.getState());

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d94d37af/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index e4a1040..41b500f 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -413,11 +413,12 @@ public class TestVertexImpl {
     vertexIdMap = new HashMap<TezVertexID, VertexImpl>();
     for (int i = 0; i < vCnt; ++i) {
       VertexPlan vPlan = dagPlan.getVertex(i);
+      String vName = vPlan.getName();
       TezVertexID vertexId = new TezVertexID(dagId, i+1);
       VertexImpl v = new VertexImpl(vertexId, vPlan, vPlan.getName(), conf,
           dispatcher.getEventHandler(), taskAttemptListener, jobToken, fsTokens,
           clock, thh, appContext, vertexLocationHint);
-      vertices.put(vPlan.getName(), v);
+      vertices.put(vName, v);
       vertexIdMap.put(vertexId, v);
     }
   }
@@ -492,16 +493,29 @@ public class TestVertexImpl {
 
   @After
   public void teardown() {
+    dispatcher.await();
+    dispatcher.stop();
+    dispatcher = null;
+    vertexEventDispatcher = null;
+    dagEventDispatcher = null;
     dagPlan = null;
     this.vertices = null;
     this.edges = null;
-    dispatcher.await();
-    dispatcher.stop();
+    this.vertexIdMap = null;
   }
 
+  private void initAllVertices() {
+    for (int i = 1; i <= 6; ++i) {
+      VertexImpl v = vertices.get("vertex" + i);
+      initVertex(v);
+    }
+  }
+
+
+  @SuppressWarnings("unchecked")
   private void initVertex(VertexImpl v) {
     Assert.assertEquals(VertexState.NEW, v.getState());
-    v.handle(new VertexEvent(v.getVertexId(),
+    dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(),
           VertexEventType.V_INIT));
     dispatcher.await();
     Assert.assertEquals(VertexState.INITED, v.getState());
@@ -511,8 +525,10 @@ public class TestVertexImpl {
     startVertex(v, true);
   }
 
+  @SuppressWarnings("unchecked")
   private void killVertex(VertexImpl v, boolean checkKillWait) {
-    v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+    dispatcher.getEventHandler().handle(
+        new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
     dispatcher.await();
     if (checkKillWait) {
       Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
@@ -521,10 +537,11 @@ public class TestVertexImpl {
     }
   }
 
+  @SuppressWarnings("unchecked")
   private void startVertex(VertexImpl v,
       boolean checkRunningState) {
     Assert.assertEquals(VertexState.INITED, v.getState());
-    v.handle(new VertexEvent(v.getVertexId(),
+    dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(),
           VertexEventType.V_START));
     dispatcher.await();
     if (checkRunningState) {
@@ -532,7 +549,7 @@ public class TestVertexImpl {
     }
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testVertexInit() {
     VertexImpl v = vertices.get("vertex2");
     initVertex(v);
@@ -583,66 +600,79 @@ public class TestVertexImpl {
             .getOutputClassName()));
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testVertexStart() {
+    initAllVertices();
+
     VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
     startVertex(v);
   }
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   public void testBasicVertexCompletion() {
+    initAllVertices();
+
     VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
     startVertex(v);
 
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
     TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.RUNNING, v.getState());
     Assert.assertEquals(1, v.getCompletedTasks());
 
-    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
     Assert.assertEquals(2, v.getCompletedTasks());
   }
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   @Ignore // FIXME fix verteximpl for this test to work
   public void testDuplicateTaskCompletion() {
+    initAllVertices();
+
     VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
     startVertex(v);
 
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
     TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.RUNNING, v.getState());
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.RUNNING, v.getState());
 
-    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
   }
 
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   public void testVertexFailure() {
+    initAllVertices();
+
     VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
     startVertex(v);
 
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.FAILED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.FAILED));
     dispatcher.await();
     Assert.assertEquals(VertexState.FAILED, v.getState());
     String diagnostics =
@@ -650,17 +680,18 @@ public class TestVertexImpl {
     Assert.assertTrue(diagnostics.contains("task failed " + t1.toString()));
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testVertexWithNoTasks() {
     // FIXME a vertex with no tasks should not be allowed
+    initAllVertices();
+
     VertexImpl v = vertices.get("vertex1");
-    initVertex(v);
     startVertex(v, false);
     dispatcher.await();
     Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testVertexKillDiagnostics() {
     VertexImpl v1 = vertices.get("vertex1");
     killVertex(v1, false);
@@ -679,7 +710,14 @@ public class TestVertexImpl {
         "vertex received kill in inited state"));
 
     VertexImpl v3 = vertices.get("vertex3");
+    VertexImpl v4 = vertices.get("vertex4");
+    VertexImpl v5 = vertices.get("vertex5");
+    VertexImpl v6 = vertices.get("vertex6");
     initVertex(v3);
+    initVertex(v4);
+    initVertex(v5);
+    initVertex(v6);
+
     startVertex(v3);
     killVertex(v3, true);
     diagnostics =
@@ -688,68 +726,77 @@ public class TestVertexImpl {
         "vertex received kill while in running state"));
   }
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   public void testVertexKillPending() {
-    VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
-    VertexImpl v3 = vertices.get("vertex3");
-    initVertex(v3);
+    initAllVertices();
 
+    VertexImpl v = vertices.get("vertex2");
     startVertex(v);
 
-    v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+    dispatcher.getEventHandler().handle(
+        new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+    dispatcher.await();
     Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
 
-    v.handle(new VertexEventTaskCompleted(
-        new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(
+            new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
+    dispatcher.await();
     Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
 
-    v.handle(new VertexEventTaskCompleted(
-        new TezTaskID(v.getVertexId(), 1), TaskState.KILLED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(
+            new TezTaskID(v.getVertexId(), 1), TaskState.KILLED));
     dispatcher.await();
     Assert.assertEquals(VertexState.KILLED, v.getState());
   }
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   @Ignore
   public void testVertexKill() {
-    VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
-    VertexImpl v3 = vertices.get("vertex3");
-    initVertex(v3);
+    initAllVertices();
 
+    VertexImpl v = vertices.get("vertex2");
     startVertex(v);
 
-    v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+    dispatcher.getEventHandler().handle(
+        new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
     Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
 
-    v.handle(new VertexEventTaskCompleted(
-        new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(
+            new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
     Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
 
-    v.handle(new VertexEventTaskCompleted(
-        new TezTaskID(v.getVertexId(), 1), TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(
+            new TezTaskID(v.getVertexId(), 1), TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.KILLED, v.getState());
   }
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   @Ignore
   public void testKilledTasksHandling() {
+    initAllVertices();
+
     VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
     startVertex(v);
 
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
     TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.FAILED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.FAILED));
     dispatcher.await();
     Assert.assertEquals(VertexState.FAILED, v.getState());
     Assert.assertEquals(TaskState.KILLED, v.getTask(t2).getState());
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testVertexCommitterInit() {
     VertexImpl v2 = vertices.get("vertex2");
     initVertex(v2);
@@ -762,7 +809,7 @@ public class TestVertexImpl {
         instanceof MRVertexOutputCommitter);
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testVertexSchedulerInit() {
     VertexImpl v2 = vertices.get("vertex2");
     initVertex(v2);
@@ -775,10 +822,13 @@ public class TestVertexImpl {
         instanceof BipartiteSlowStartVertexScheduler);
   }
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   public void testVertexTaskFailure() {
+    initAllVertices();
+
     VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
+
     CountingVertexOutputCommitter committer =
         new CountingVertexOutputCommitter();
     v.setVertexOutputCommitter(committer);
@@ -787,62 +837,66 @@ public class TestVertexImpl {
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
     TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.RUNNING, v.getState());
 
-    v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
-    v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t2, TaskState.FAILED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t2, TaskState.FAILED));
     dispatcher.await();
     Assert.assertEquals(VertexState.FAILED, v.getState());
     Assert.assertEquals(0, committer.commitCounter);
     Assert.assertEquals(1, committer.abortCounter);
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testSourceVertexStartHandling() {
+    LOG.info("Testing testSourceVertexStartHandling");
+    initAllVertices();
+
     VertexImpl v4 = vertices.get("vertex4");
-    initVertex(v4);
     VertexImpl v5 = vertices.get("vertex5");
-    initVertex(v5);
     VertexImpl v6 = vertices.get("vertex6");
-    initVertex(v6);
-    Assert.assertEquals(VertexState.INITED, v6.getState());
 
     startVertex(v4);
     startVertex(v5);
     dispatcher.await();
+    LOG.info("Verifying v6 state " + v6.getState());
     Assert.assertEquals(VertexState.RUNNING, v6.getState());
     Assert.assertEquals(1, v6.getDistanceFromRoot());
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testCounters() {
     // FIXME need to test counters at vertex level
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testDiagnostics() {
     // FIXME need to test diagnostics in various cases
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testTaskAttemptCompletionEvents() {
     // FIXME need to test handling of task attempt events
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testSourceTaskAttemptCompletionEvents() {
+    LOG.info("Testing testSourceTaskAttemptCompletionEvents");
+    initAllVertices();
+
     VertexImpl v4 = vertices.get("vertex4");
-    initVertex(v4);
     VertexImpl v5 = vertices.get("vertex5");
-    initVertex(v5);
     VertexImpl v6 = vertices.get("vertex6");
-    initVertex(v6);
 
     startVertex(v4);
     startVertex(v5);
     dispatcher.await();
+    LOG.info("Verifying v6 state " + v6.getState());
     Assert.assertEquals(VertexState.RUNNING, v6.getState());
 
     TezTaskID t1_v4 = new TezTaskID(v4.getVertexId(), 0);
@@ -897,17 +951,21 @@ public class TestVertexImpl {
     Assert.assertEquals(6, v6.getTaskAttemptCompletionEvents(0, 100).length);
   }
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   public void testDAGEventGeneration() {
+    initAllVertices();
+
     VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
     startVertex(v);
 
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
     TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
-    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
     Assert.assertEquals(1,
@@ -915,11 +973,14 @@ public class TestVertexImpl {
             DAGEventType.DAG_VERTEX_COMPLETED).intValue());
   }
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   public void testTaskReschedule() {
     // For downstream failures
+    initAllVertices();
+
     VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
+
     CountingVertexOutputCommitter committer =
         new CountingVertexOutputCommitter();
     v.setVertexOutputCommitter(committer);
@@ -929,26 +990,33 @@ public class TestVertexImpl {
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
     TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
-    v.handle(new VertexEventTaskReschedule(t1));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskReschedule(t1));
     // FIXME need to handle dups
-    // v.handle(new VertexEventTaskReschedule(t1));
-    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    // dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.RUNNING, v.getState());
     Assert.assertEquals(0, committer.commitCounter);
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
     Assert.assertEquals(1, committer.commitCounter);
 
   }
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   public void testVertexCommit() {
+    initAllVertices();
+
     VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
+
     CountingVertexOutputCommitter committer =
         new CountingVertexOutputCommitter();
     v.setVertexOutputCommitter(committer);
@@ -958,14 +1026,18 @@ public class TestVertexImpl {
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
     TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
-    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
     Assert.assertEquals(1, committer.commitCounter);
 
-    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
     Assert.assertEquals(1, committer.commitCounter);
@@ -974,20 +1046,23 @@ public class TestVertexImpl {
     Assert.assertEquals(0, committer.setupCounter); // already done in init
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testCommitterInitAndSetup() {
     // FIXME need to add a test for this
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testTaskAttemptFetchFailureHandling() {
     // FIXME needs testing
   }
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   public void testBadCommitter() {
+    initAllVertices();
+
     VertexImpl v = vertices.get("vertex2");
-    initVertex(v);
+
     CountingVertexOutputCommitter committer =
         new CountingVertexOutputCommitter(true, true);
     v.setVertexOutputCommitter(committer);
@@ -997,8 +1072,10 @@ public class TestVertexImpl {
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
     TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
-    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.getEventHandler().handle(
+        new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.FAILED, v.getState());
     Assert.assertEquals(1, committer.commitCounter);
@@ -1009,14 +1086,15 @@ public class TestVertexImpl {
     Assert.assertEquals(0, committer.setupCounter); // already done in init
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testHistoryEventGeneration() {
   }
 
-  @Test
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   public void testInvalidEvent() {
     VertexImpl v = vertices.get("vertex2");
-    v.handle(new VertexEvent(v.getVertexId(),
+    dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(),
         VertexEventType.V_START));
     dispatcher.await();
     Assert.assertEquals(VertexState.ERROR, v.getState());


[49/50] [abbrv] git commit: TEZ-179. Update JobContextImpl to use the changed return type for certain methods post MAPREDUCE-5300. (sseth)

Posted by ss...@apache.org.
TEZ-179. Update JobContextImpl to use the changed return type for
certain methods post MAPREDUCE-5300. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a10e607a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a10e607a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a10e607a

Branch: refs/heads/master
Commit: a10e607ac5f3eeae71df8dc32c08b95c3a2ebd82
Parents: 114b12a
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jun 3 22:06:03 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jun 3 22:06:03 2013 -0700

----------------------------------------------------------------------
 .../mapreduce/hadoop/mapreduce/JobContextImpl.java |   24 +++++++++++++--
 1 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a10e607a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
index 1f34fdb..4962260 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
@@ -352,7 +352,25 @@ public class JobContextImpl implements JobContext {
   public Path[] getFileClassPaths() {
     return DistributedCache.getFileClassPaths(conf);
   }
-  
+
+  /**
+   * Parse a list of longs into strings.
+   * 
+   * @param timestamps
+   *          the list of longs to parse
+   * @return a list of string that were parsed. same length as timestamps.
+   */
+  private static String[] toTimestampStrs(long[] timestamps) {
+    if (timestamps == null) {
+      return null;
+    }
+    String[] result = new String[timestamps.length];
+    for (int i = 0; i < timestamps.length; ++i) {
+      result[i] = Long.toString(timestamps[i]);
+    }
+    return result;
+  }
+
   /**
    * Get the timestamps of the archives.  Used by internal
    * DistributedCache and MapReduce code.
@@ -360,7 +378,7 @@ public class JobContextImpl implements JobContext {
    * @throws IOException
    */
   public String[] getArchiveTimestamps() {
-    return DistributedCache.getArchiveTimestamps(conf);
+    return toTimestampStrs(DistributedCache.getArchiveTimestamps(conf));
   }
 
   /**
@@ -370,7 +388,7 @@ public class JobContextImpl implements JobContext {
    * @throws IOException
    */
   public String[] getFileTimestamps() {
-    return DistributedCache.getFileTimestamps(conf);
+    return toTimestampStrs(DistributedCache.getFileTimestamps(conf));
   }
 
   /** 


[13/50] [abbrv] git commit: TEZ-62. Fix and enable tests in TestMapProcessor and TestReduceProcessor. (sseth)

Posted by ss...@apache.org.
TEZ-62. Fix and enable tests in TestMapProcessor and
TestReduceProcessor. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/cee79f63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/cee79f63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/cee79f63

Branch: refs/heads/master
Commit: cee79f6384940ca226cd0c2ef02e024f19e7550a
Parents: 38b410e
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri May 17 00:03:20 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Fri May 17 00:03:20 2013 -0700

----------------------------------------------------------------------
 .../java/org/apache/tez/common/TezJobConfig.java   |    7 +
 .../org/apache/hadoop/mapred/YarnTezDagChild.java  |   10 +-
 .../apache/tez/common/TezEngineTaskContext.java    |    4 +-
 .../mapreduce/split/SplitMetaInfoReaderTez.java    |   14 ++-
 .../tez/mapreduce/hadoop/TezTypeConverters.java    |    2 +
 .../org/apache/tez/mapreduce/processor/MRTask.java |    8 +-
 .../tez/mapreduce/processor/map/MapProcessor.java  |   14 ++-
 .../tez/mapreduce/TestUmbilicalProtocol.java       |    2 +-
 .../apache/tez/mapreduce/processor/MapUtils.java   |   85 ++++++++----
 .../mapreduce/processor/map/TestMapProcessor.java  |   79 +++++++++---
 .../processor/reduce/TestReduceProcessor.java      |  104 +++++++++++----
 11 files changed, 236 insertions(+), 93 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
index e867458..4a7abea 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -68,6 +68,13 @@ public class TezJobConfig {
   public static final String DEFAULT_LOCAL_DIR = "/tmp";
 
   /**
+   * The directory which contains the localized files for this task.
+   */
+  @Private
+  public static final String TASK_LOCAL_RESOURCE_DIR = "tez.engine.task-local-resource.dir";
+  public static final String DEFAULT_TASK_LOCAL_RESOURCE_DIR = "/tmp";
+  
+  /**
    * 
    */
   public static final String TEZ_ENGINE_TASK_INDEGREE = 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index f140c3a..2295b6a 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -249,12 +249,16 @@ public class YarnTezDagChild {
    * out an output directory.
    * @throws IOException 
    */
-  private static void configureLocalDirs(MRTask task, JobConf job) throws IOException {
+  private static void configureLocalDirs(JobConf job) throws IOException {
     String[] localSysDirs = StringUtils.getTrimmedStrings(
         System.getenv(Environment.LOCAL_DIRS.name()));
     job.setStrings(TezJobConfig.LOCAL_DIR, localSysDirs);
+    job.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
+        System.getenv(Environment.PWD.name()));
     LOG.info(TezJobConfig.LOCAL_DIR + " for child: " +
         job.get(TezJobConfig.LOCAL_DIR));
+    LOG.info(TezJobConfig.TASK_LOCAL_RESOURCE_DIR + " for child: "
+        + job.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
     LocalDirAllocator lDirAlloc = new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
     Path workDir = null;
     // First, try to find the JOB_LOCAL_DIR on this host.
@@ -285,7 +289,7 @@ public class YarnTezDagChild {
       }
     }
     // TODO TEZ This likely needs fixing to make sure things work when there are multiple local-dirs etc.
-    job.set(MRJobConfig.JOB_LOCAL_DIR,workDir.toString());
+    job.set(MRJobConfig.JOB_LOCAL_DIR, workDir.toString());
   }
 
   private static JobConf configureTask(MRTask task, Credentials credentials,
@@ -308,7 +312,7 @@ public class YarnTezDagChild {
 //        JobTokenSecretManager.createSecretKey(jt.getPassword()));
 
     // setup the child's MRConfig.LOCAL_DIR.
-    configureLocalDirs(task, job);
+    configureLocalDirs(job);
 
     // setup the child's attempt directories
     // Do the task-type specific localization

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
index 350702a..5823de6 100644
--- a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
+++ b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
@@ -39,7 +39,7 @@ public class TezEngineTaskContext extends TezTaskContext {
   }
 
   public TezEngineTaskContext(TezTaskAttemptID taskAttemptID, String user,
-      String jobName, String vertexName, String moduleClassName,
+      String jobName, String vertexName, String processorName,
       List<InputSpec> inputSpecList, List<OutputSpec> outputSpecList) {
     super(taskAttemptID, user, jobName, vertexName);
     this.inputSpecList = inputSpecList;
@@ -52,7 +52,7 @@ public class TezEngineTaskContext extends TezTaskContext {
     }
     this.inputSpecList = inputSpecList;
     this.outputSpecList = outputSpecList;
-    this.processorName = moduleClassName;
+    this.processorName = processorName;
   }
 
   public String getProcessorName() {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
index de8d972..72246c0 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
+import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 /**
@@ -44,6 +45,9 @@ public class SplitMetaInfoReaderTez {
 
   public static final Log LOG = LogFactory.getLog(SplitMetaInfoReaderTez.class);
 
+  public static final int META_SPLIT_VERSION = JobSplit.META_SPLIT_VERSION;
+  public static final byte[] META_SPLIT_FILE_HEADER = JobSplit.META_SPLIT_FILE_HEADER;
+
   // Forked from the MR variant so that the metaInfo file as well as the split
   // file can be read from local fs - relying on these files being localized.
   public static TaskSplitMetaInfo[] readSplitMetaInfo(Configuration conf,
@@ -53,9 +57,11 @@ public class SplitMetaInfoReaderTez {
         MRJobConfig.SPLIT_METAINFO_MAXSIZE,
         MRJobConfig.DEFAULT_SPLIT_METAINFO_MAXSIZE);
 
-    Path metaSplitFile = new Path(MRJobConfig.JOB_SPLIT_METAINFO);
+    Path metaSplitFile = new Path(
+        conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR),
+        MRJobConfig.JOB_SPLIT_METAINFO);
     String jobSplitFile = MRJobConfig.JOB_SPLIT;
-
+    
     File file = new File(metaSplitFile.toUri().getPath()).getAbsoluteFile();
     LOG.info("DEBUG: Setting up JobSplitIndex with JobSplitFile at: "
         + file.getAbsolutePath() + ", defaultFS from conf: "
@@ -83,12 +89,12 @@ public class SplitMetaInfoReaderTez {
       JobSplit.SplitMetaInfo splitMetaInfo = new JobSplit.SplitMetaInfo();
       splitMetaInfo.readFields(in);
       JobSplit.TaskSplitIndex splitIndex = new JobSplit.TaskSplitIndex(
-          jobSplitFile, splitMetaInfo.getStartOffset());
+          new Path(conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR), jobSplitFile)
+              .toUri().toString(), splitMetaInfo.getStartOffset());
       allSplitMetaInfo[i] = new JobSplit.TaskSplitMetaInfo(splitIndex,
           splitMetaInfo.getLocations(), splitMetaInfo.getInputDataLength());
     }
     in.close();
     return allSplitMetaInfo;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
index 1fc71b0..768d347 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
@@ -34,6 +34,8 @@ import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
 
 public class TezTypeConverters {
 
+  // TODO Remove unused methods
+  
   // Tez objects will be imported. Others will use the fully qualified name when
   // required.
   // All public methods named toYarn / toTez / toMapReduce

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
index d7d721f..276294c 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
@@ -149,7 +149,7 @@ extends RunningTaskContext {
     
     jobConf.set(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID, 
         taskAttemptId.toString());
-    
+
     initResourceCalculatorPlugin();
     
     LOG.info("MRTask.inited: taskAttemptId = " + taskAttemptId.toString());
@@ -695,8 +695,10 @@ extends RunningTaskContext {
 
   public void localizeConfiguration(JobConf jobConf) 
       throws IOException, InterruptedException {
-    jobConf.set(JobContext.TASK_ID, taskAttemptId.getTaskID().toString()); 
-    jobConf.set(JobContext.TASK_ATTEMPT_ID, taskAttemptId.toString());
+    jobConf.set(JobContext.TASK_ID, IDConverter
+        .toMRTaskAttemptId(taskAttemptId).toString());
+    jobConf.set(JobContext.TASK_ATTEMPT_ID,
+        IDConverter.toMRTaskAttemptId(taskAttemptId).toString());
     jobConf.setInt(JobContext.TASK_PARTITION, 
         taskAttemptId.getTaskID().getId());
     jobConf.set(JobContext.ID, taskAttemptId.getTaskID().getVertexID().getDAGId().toString());

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
index a6ab986..3558739 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
@@ -69,11 +69,7 @@ public class MapProcessor extends MRTask implements Processor {
   public void initialize(Configuration conf, Master master) throws IOException,
   InterruptedException {
     super.initialize(conf, master);
-    TaskSplitMetaInfo[] allMetaInfo = readSplits();
-    TaskSplitMetaInfo thisTaskMetaInfo = allMetaInfo[tezEngineTaskContext
-        .getTaskAttemptId().getTaskID().getId()];
-    splitMetaInfo = new TaskSplitIndex(thisTaskMetaInfo.getSplitLocation(),
-        thisTaskMetaInfo.getStartOffset());
+    
   }
 
   @Override
@@ -81,6 +77,14 @@ public class MapProcessor extends MRTask implements Processor {
       final Input[] ins,
       final Output[] outs)
           throws IOException, InterruptedException {
+    
+    // Read split information.
+    TaskSplitMetaInfo[] allMetaInfo = readSplits();
+    TaskSplitMetaInfo thisTaskMetaInfo = allMetaInfo[tezEngineTaskContext
+        .getTaskAttemptId().getTaskID().getId()];
+    splitMetaInfo = new TaskSplitIndex(thisTaskMetaInfo.getSplitLocation(),
+        thisTaskMetaInfo.getStartOffset());
+    
     MRTaskReporter reporter = new MRTaskReporter(getTaskReporter());
     boolean useNewApi = jobConf.getUseNewMapper();
     initTask(jobConf, taskAttemptId.getTaskID().getVertexID().getDAGId(),

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
index 025bf61..840cb31 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
@@ -111,7 +111,7 @@ public class TestUmbilicalProtocol implements TezTaskUmbilicalProtocol {
   @Override
   public boolean canCommit(TezTaskAttemptID taskid) throws IOException {
     LOG.info("Got 'can-commit' from " + taskid);
-    return false;
+    return true;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
index aec6667..0b6bc5f 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
@@ -18,40 +18,37 @@
 
 package org.apache.tez.mapreduce.processor;
 
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
-
 import java.io.IOException;
-import java.util.Collections;
+import java.util.List;
 import java.util.Random;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.FileSplit;
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex;
+import org.apache.hadoop.mapreduce.split.JobSplit;
+import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo;
+import org.apache.hadoop.mapreduce.split.SplitMetaInfoReaderTez;
 import org.apache.tez.common.InputSpec;
 import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.TezEngineTaskContext;
+import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
-import org.apache.tez.engine.api.Input;
-import org.apache.tez.engine.api.Output;
-import org.apache.tez.engine.api.Processor;
 import org.apache.tez.engine.api.Task;
-import org.apache.tez.engine.lib.output.LocalOnFileSorterOutput;
 import org.apache.tez.engine.runtime.RuntimeUtils;
-import org.apache.tez.engine.task.RuntimeTask;
 import org.apache.tez.mapreduce.TezTestUtils;
-import org.apache.tez.mapreduce.input.SimpleInput;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 import org.apache.tez.mapreduce.processor.map.MapProcessor;
 
 public class MapUtils {
@@ -62,8 +59,7 @@ public class MapUtils {
   createInputSplit(FileSystem fs, Path workDir, JobConf job, Path file) 
       throws IOException {
     FileInputFormat.setInputPaths(job, workDir);
-  
-    
+
     // create a file with length entries
     SequenceFile.Writer writer = 
         SequenceFile.createWriter(fs, job, file, 
@@ -92,30 +88,59 @@ public class MapUtils {
         "file = " + ((FileSplit)splits[0]).getPath());
     return splits[0];
   }
+  
+  final private static FsPermission JOB_FILE_PERMISSION = FsPermission
+      .createImmutable((short) 0644); // rw-r--r--
+
+  // Will write files to PWD, from where they are read.
+  
+  private static void writeSplitFiles(FileSystem fs, JobConf conf,
+      InputSplit split) throws IOException {
+    Path jobSplitFile = new Path(conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
+        TezJobConfig.DEFAULT_TASK_LOCAL_RESOURCE_DIR), MRJobConfig.JOB_SPLIT);
+    FSDataOutputStream out = FileSystem.create(fs, jobSplitFile,
+        new FsPermission(JOB_FILE_PERMISSION));
+
+    long offset = out.getPos();
+    Text.writeString(out, split.getClass().getName());
+    split.write(out);
+    out.close();
+
+    String[] locations = split.getLocations();
+
+    SplitMetaInfo info = null;
+    info = new JobSplit.SplitMetaInfo(locations, offset, split.getLength());
+
+    Path jobSplitMetaInfoFile = new Path(
+        conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR),
+        MRJobConfig.JOB_SPLIT_METAINFO);
+
+    FSDataOutputStream outMeta = FileSystem.create(fs, jobSplitMetaInfoFile,
+        new FsPermission(JOB_FILE_PERMISSION));
+    outMeta.write(SplitMetaInfoReaderTez.META_SPLIT_FILE_HEADER);
+    WritableUtils.writeVInt(outMeta, SplitMetaInfoReaderTez.META_SPLIT_VERSION);
+    WritableUtils.writeVInt(outMeta, 1); // Only 1 split meta info being written
+    info.write(outMeta);
+    outMeta.close();
+  }
 
   public static Task runMapProcessor(FileSystem fs, Path workDir,
-      JobConf jobConf,
-      int mapId, Path mapInput,
+      JobConf jobConf, int mapId, Path mapInput,
       TezTaskUmbilicalProtocol umbilical,
-      Class<?> outputClazz) throws Exception {
+      String vertexName, List<InputSpec> inputSpecs,
+      List<OutputSpec> outputSpecs) throws Exception {
     jobConf.setInputFormat(SequenceFileInputFormat.class);
     InputSplit split = createInputSplit(fs, workDir, jobConf, mapInput);
-    TezEngineTaskContext taskContext = 
-        new TezEngineTaskContext(
-        TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0), "tez",
-        "tez", "TODO_vertexName", MapProcessor.class.getName(),
-        Collections.singletonList(new InputSpec("srcVertex", 0,
-            SimpleInput.class.getName())),
-        Collections.singletonList(new OutputSpec("targetVertex", 0,
-            outputClazz.getName())));
+
+    writeSplitFiles(fs, jobConf, split);
+    TezEngineTaskContext taskContext = new TezEngineTaskContext(
+        TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0), "testuser",
+        "testJob", vertexName, MapProcessor.class.getName(),
+        inputSpecs, outputSpecs);
 
     Task t = RuntimeUtils.createRuntimeTask(taskContext);
     t.initialize(jobConf, umbilical);
-    SimpleInput[] real = ((SimpleInput[])t.getInputs());
-    SimpleInput[] inputs = spy(real);
-    doReturn(split).when(inputs[0]).getOldSplitDetails(any(TaskSplitIndex.class));
-    t.getProcessor().process(inputs, t.getOutputs());
+    t.getProcessor().process(t.getInputs(), t.getOutputs());
     return t;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
index f09340c..cda15fb 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
@@ -19,9 +19,11 @@ package org.apache.tez.mapreduce.processor.map;
 
 
 import java.io.IOException;
+import java.util.Collections;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputBuffer;
@@ -29,6 +31,8 @@ import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.tez.common.Constants;
+import org.apache.tez.common.InputSpec;
+import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.engine.api.Task;
 import org.apache.tez.engine.common.sort.impl.IFile;
@@ -37,11 +41,15 @@ import org.apache.tez.engine.common.task.local.output.TezTaskOutput;
 import org.apache.tez.engine.lib.output.InMemorySortedOutput;
 import org.apache.tez.engine.lib.output.LocalOnFileSorterOutput;
 import org.apache.tez.mapreduce.TestUmbilicalProtocol;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfToTezTranslator;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+import org.apache.tez.mapreduce.input.SimpleInput;
 import org.apache.tez.mapreduce.processor.MapUtils;
 import org.jboss.netty.buffer.BigEndianHeapChannelBuffer;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.buffer.TruncatedChannelBuffer;
 import org.jboss.netty.handler.stream.ChunkedStream;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -49,9 +57,7 @@ import org.junit.Test;
 
 public class TestMapProcessor {
   
-  private static final Log LOG = LogFactory.getLog(TestMapProcessor.class);
-  
-  JobConf job;
+  private static final Log LOG = LogFactory.getLog(TestMapProcessor.class);  
   
   private static JobConf defaultConf = new JobConf();
   private static FileSystem localFs = null; 
@@ -63,32 +69,51 @@ public class TestMapProcessor {
       throw new RuntimeException("init failure", e);
     }
   }
+  @SuppressWarnings("deprecation")
   private static Path workDir =
     new Path(new Path(System.getProperty("test.build.data", "/tmp")),
              "TestMapProcessor").makeQualified(localFs);
 
   TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
-  
-  @Before
-  public void setUp() {
-    job = new JobConf(defaultConf);
+
+  public void setUpJobConf(JobConf job) {
     job.set(TezJobConfig.LOCAL_DIR, workDir.toString());
     job.setClass(
         Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
         TezLocalTaskOutputFiles.class, 
         TezTaskOutput.class);
     job.setNumReduceTasks(1);
-    mapOutputs.setConf(job);
+  }
+  
+  @Before
+  @After
+  public void cleanup() throws Exception {
+    localFs.delete(workDir, true);
   }
   
   @Test
-  @Ignore
   public void testMapProcessor() throws Exception {
-    localFs.delete(workDir, true);
-    MapUtils.runMapProcessor(
-        localFs, workDir, job, 0, new Path(workDir, "map0"), 
-        new TestUmbilicalProtocol(),
-        LocalOnFileSorterOutput.class).close();
+    String vertexName = MultiStageMRConfigUtil.getInitialMapVertexName();
+    JobConf jobConf = new JobConf(defaultConf);
+    setUpJobConf(jobConf);
+    TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
+    mapOutputs.setConf(jobConf);
+    
+    Configuration conf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
+    Configuration stageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
+        vertexName);
+    
+    JobConf job = new JobConf(stageConf);
+    
+    job.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
+        "localized-resources").toUri().toString());
+
+    MapUtils.runMapProcessor(localFs, workDir, job, 0,
+        new Path(workDir, "map0"), new TestUmbilicalProtocol(), vertexName,
+        Collections.singletonList(new InputSpec("NullVertex", 0,
+            SimpleInput.class.getName())),
+        Collections.singletonList(new OutputSpec("FakeVertex", 1,
+            LocalOnFileSorterOutput.class.getName()))).close();
 
     Path mapOutputFile = mapOutputs.getInputFile(0);
     LOG.info("mapOutputFile = " + mapOutputFile);
@@ -115,16 +140,34 @@ public class TestMapProcessor {
   @Test
   @Ignore
   public void testMapProcessorWithInMemSort() throws Exception {
+    
+    String vertexName = MultiStageMRConfigUtil.getInitialMapVertexName();
+    
     final int partitions = 2;
-    job.setNumReduceTasks(partitions);
-    job.setInt(TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE, partitions); 
+    JobConf jobConf = new JobConf(defaultConf);
+    jobConf.setNumReduceTasks(partitions);
+    setUpJobConf(jobConf);
+    TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
+    mapOutputs.setConf(jobConf);
+    
+    Configuration conf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
+    Configuration stageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
+        vertexName);
+    
+    JobConf job = new JobConf(stageConf);
 
+    job.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
+        "localized-resources").toUri().toString());
     localFs.delete(workDir, true);
     Task t =
         MapUtils.runMapProcessor(
             localFs, workDir, job, 0, new Path(workDir, "map0"), 
-            new TestUmbilicalProtocol(true),
-            InMemorySortedOutput.class);
+            new TestUmbilicalProtocol(true), vertexName, 
+            Collections.singletonList(new InputSpec("NullVertex", 0,
+                SimpleInput.class.getName())),
+            Collections.singletonList(new OutputSpec("FakeVertex", 1,
+                InMemorySortedOutput.class.getName()))
+            );
     InMemorySortedOutput[] outputs = (InMemorySortedOutput[])t.getOutputs();
     
     verifyInMemSortedStream(outputs[0], 0, 4096);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
index afc21a7..69571e1 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
@@ -22,40 +22,44 @@ import java.util.Collections;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.FileOutputFormat;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.tez.common.Constants;
 import org.apache.tez.common.InputSpec;
 import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.TezEngineTaskContext;
 import org.apache.tez.common.TezJobConfig;
-import org.apache.tez.engine.api.Input;
-import org.apache.tez.engine.api.Output;
 import org.apache.tez.engine.api.Task;
 import org.apache.tez.engine.common.task.local.output.TezLocalTaskOutputFiles;
 import org.apache.tez.engine.common.task.local.output.TezTaskOutput;
 import org.apache.tez.engine.lib.input.LocalMergedInput;
 import org.apache.tez.engine.lib.output.LocalOnFileSorterOutput;
 import org.apache.tez.engine.runtime.RuntimeUtils;
-import org.apache.tez.engine.task.RuntimeTask;
 import org.apache.tez.mapreduce.TestUmbilicalProtocol;
 import org.apache.tez.mapreduce.TezTestUtils;
+import org.apache.tez.mapreduce.hadoop.IDConverter;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfToTezTranslator;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+import org.apache.tez.mapreduce.input.SimpleInput;
 import org.apache.tez.mapreduce.output.SimpleOutput;
 import org.apache.tez.mapreduce.processor.MapUtils;
+import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 
 public class TestReduceProcessor {
   
   private static final Log LOG = LogFactory.getLog(TestReduceProcessor.class);
-  
-  JobConf job;
-  
+
   private static JobConf defaultConf = new JobConf();
   private static FileSystem localFs = null; 
   static {
@@ -66,13 +70,12 @@ public class TestReduceProcessor {
       throw new RuntimeException("init failure", e);
     }
   }
+  @SuppressWarnings("deprecation")
   private static Path workDir =
     new Path(new Path(System.getProperty("test.build.data", "/tmp")),
              "TestReduceProcessor").makeQualified(localFs);
 
-  @Before
-  public void setUp() {
-    job = new JobConf(defaultConf);
+  public void setUpJobConf(JobConf job) {
     job.set(TezJobConfig.LOCAL_DIR, workDir.toString());
     job.setClass(
         Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
@@ -80,37 +83,84 @@ public class TestReduceProcessor {
         TezTaskOutput.class);
     job.setNumReduceTasks(1);
   }
-  
+
+  @Before
+  @After
+  public void cleanup() throws Exception {
+    localFs.delete(workDir, true);
+  }
+
   @Test
-  @Ignore
   public void testReduceProcessor() throws Exception {
-    localFs.delete(workDir, true);
 
+    String mapVertexName = MultiStageMRConfigUtil.getInitialMapVertexName();
+    String reduceVertexName = MultiStageMRConfigUtil.getFinalReduceVertexName();
+    JobConf jobConf = new JobConf(defaultConf);
+    setUpJobConf(jobConf);
+    TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
+    mapOutputs.setConf(jobConf);
+    
+    Configuration conf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
+    Configuration mapStageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
+        mapVertexName);
+    
+    JobConf mapConf = new JobConf(mapStageConf);
+    
+    mapConf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
+        "localized-resources").toUri().toString());
+    
     // Run a map
-    MapUtils.runMapProcessor(
-        localFs, workDir, job, 0, new Path(workDir, "map0"), 
-        new TestUmbilicalProtocol(),
-        LocalOnFileSorterOutput.class);
+    MapUtils.runMapProcessor(localFs, workDir, mapConf, 0,
+        new Path(workDir, "map0"), new TestUmbilicalProtocol(), mapVertexName,
+        Collections.singletonList(new InputSpec("NullVertex", 0,
+            SimpleInput.class.getName())),
+        Collections.singletonList(new OutputSpec("FakeVertex", 1,
+            LocalOnFileSorterOutput.class.getName())));
 
     LOG.info("Starting reduce...");
-    FileOutputFormat.setOutputPath(job, new Path(workDir, "output"));
+    
+    
+    Configuration reduceStageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
+        reduceVertexName);
+    JobConf reduceConf = new JobConf(reduceStageConf);
+    reduceConf.setOutputFormat(SequenceFileOutputFormat.class);
+    FileOutputFormat.setOutputPath(reduceConf, new Path(workDir, "output"));
     
     // Now run a reduce
     TezEngineTaskContext taskContext = new TezEngineTaskContext(
-        TezTestUtils.getMockTaskAttemptId(0, 0, 0, 0), "tez",
-        "tez", "TODO_vertexName", ReduceProcessor.class.getName(),
-        Collections.singletonList(new InputSpec("TODO_srcVertexName", 1,
+        TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0), "testUser",
+        "testJob", reduceVertexName, ReduceProcessor.class.getName(),
+        Collections.singletonList(new InputSpec(mapVertexName, 1,
             LocalMergedInput.class.getName())),
-        Collections.singletonList(new OutputSpec("TODO_targetVertexName", 1,
+        Collections.singletonList(new OutputSpec("", 1,
                 SimpleOutput.class.getName())));
-            
-    job.set(JobContext.TASK_ATTEMPT_ID,
-        taskContext.getTaskAttemptId().toString());
+    
     Task t = RuntimeUtils.createRuntimeTask(taskContext);
-    t.initialize(job, new TestUmbilicalProtocol());
+    t.initialize(reduceConf, new TestUmbilicalProtocol());
     t.run();
     t.close();
+    
+    // Can this be done via some utility class ? MapOutputFile derivative, or
+    // instantiating the OutputCommitter
+    Path reduceOutputDir = new Path(new Path(workDir, "output"),
+        "_temporary/0/" + IDConverter
+            .toMRTaskId(taskContext.getTaskAttemptId().getTaskID()));
+    Path reduceOutputFile = new Path(reduceOutputDir, "part-00000");
+
+    @SuppressWarnings("deprecation")
+    SequenceFile.Reader reader = new SequenceFile.Reader(localFs, reduceOutputFile, reduceConf);
+
+    LongWritable key = new LongWritable();
+    Text value = new Text();
+    long prev = Long.MIN_VALUE;
+    while (reader.next(key, value)) {
+      if (prev != Long.MIN_VALUE) {
+        Assert.assertTrue(prev < key.get());
+        prev = key.get();
+      }
+    }
 
+    reader.close();
   }
 
 }


[24/50] [abbrv] git commit: TEZ-153. Fix MAPREDUCE-5156 build break (bikas)

Posted by ss...@apache.org.
TEZ-153. Fix MAPREDUCE-5156 build break (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/36198a63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/36198a63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/36198a63

Branch: refs/heads/master
Commit: 36198a636d2db65fb91934725dc6759d5dda231f
Parents: 7d401d2
Author: Bikas Saha <bi...@apache.org>
Authored: Wed May 22 16:14:43 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Wed May 22 16:14:43 2013 -0700

----------------------------------------------------------------------
 .../tez/mapreduce/examples/ExampleDriver.java      |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36198a63/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
index 0e4055a..94ba79f 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
@@ -61,7 +61,7 @@ public class ExampleDriver {
           + " containing employee_name department name per line of input"
           + " and generates count of employees per department and"
           + " sorted on employee count");
-      exitCode = pgd.driver(argv);
+      exitCode = pgd.run(argv);
     }
     catch(Throwable e){
       e.printStackTrace();


[18/50] [abbrv] git commit: TEZ-111. Create tests for DAGImpl. (hitesh)

Posted by ss...@apache.org.
TEZ-111. Create tests for DAGImpl. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/8298190d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/8298190d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/8298190d

Branch: refs/heads/master
Commit: 8298190ddb513652ea65f72a5104f6b3b3320c86
Parents: be6d4bc
Author: Hitesh Shah <hi...@apache.org>
Authored: Mon May 20 14:49:43 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Mon May 20 14:49:43 2013 -0700

----------------------------------------------------------------------
 .../apache/tez/dag/app/dag/event/DAGEventType.java |   17 +-
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |   45 +-
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |   76 +-
 .../apache/tez/dag/app/dag/impl/TestDAGImpl.java   |  626 +++++++++++++++
 .../tez/dag/app/dag/impl/TestVertexImpl.java       |  167 +++-
 5 files changed, 814 insertions(+), 117 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8298190d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
index 20f14b2..14c2f30 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
@@ -30,24 +30,17 @@ public enum DAGEventType {
   DAG_INIT,
   DAG_START,
 
-  //Producer:Task
-  /*
-  JOB_TASK_COMPLETED,
-  JOB_MAP_TASK_RESCHEDULED,
-  JOB_TASK_ATTEMPT_COMPLETED,
-  */
-  
   //Producer: Vertex
-  DAG_VERTEX_INITED,
-  DAG_VERTEX_STARTED,
   DAG_VERTEX_COMPLETED,
+
+  //Producer: TaskImpl
   DAG_SCHEDULER_UPDATE,
-  
-  //Producer:Job
+
+  //Producer:Dag
   DAG_COMPLETED,
 
   //Producer:Any component
   DAG_DIAGNOSTIC_UPDATE,
   INTERNAL_ERROR,
-  DAG_COUNTER_UPDATE,  
+  DAG_COUNTER_UPDATE,
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8298190d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 66f9c36..5f47818 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -117,7 +117,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   private final TaskAttemptListener taskAttemptListener;
   private final TaskHeartbeatHandler taskHeartbeatHandler;
   private final Object tasksSyncHandle = new Object();
-  
+
   private DAGScheduler dagScheduler;
 
   private final EventHandler eventHandler;
@@ -129,7 +129,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   private final AppContext appContext;
 
   volatile Map<TezVertexID, Vertex> vertices = new HashMap<TezVertexID, Vertex>();
-  private Map<String, EdgeProperty> edges = new HashMap<String, EdgeProperty>(); 
+  private Map<String, EdgeProperty> edges = new HashMap<String, EdgeProperty>();
   private TezCounters dagCounters = new TezCounters();
   private Object fullCountersLock = new Object();
   private TezCounters fullCounters = null;
@@ -150,7 +150,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
       INTERNAL_ERROR_TRANSITION = new InternalErrorTransition();
   private static final CounterUpdateTransition COUNTER_UPDATE_TRANSITION =
       new CounterUpdateTransition();
-  private static final DAGSchedulerUpdateTransition 
+  private static final DAGSchedulerUpdateTransition
           DAG_SCHEDULER_UPDATE_TRANSITION = new DAGSchedulerUpdateTransition();
 
   protected static final
@@ -211,8 +211,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
               DIAGNOSTIC_UPDATE_TRANSITION)
           .addTransition(DAGState.RUNNING, DAGState.RUNNING,
               DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
-          .addTransition(DAGState.RUNNING, DAGState.RUNNING, 
-              DAGEventType.DAG_SCHEDULER_UPDATE, 
+          .addTransition(DAGState.RUNNING, DAGState.RUNNING,
+              DAGEventType.DAG_SCHEDULER_UPDATE,
               DAG_SCHEDULER_UPDATE_TRANSITION)
           .addTransition(
               DAGState.RUNNING,
@@ -332,7 +332,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
     this.jobPlan = jobPlan;
     this.conf = conf;
     this.dagName = (jobPlan.getName() != null) ? jobPlan.getName() : "<missing app name>";
-    
+
     this.userName = appUserName;
     // TODO Metrics
     //this.metrics = metrics;
@@ -374,7 +374,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   public TezConfiguration getConf() {
     return conf;
   }
-  
+
   @Override
   public DAGPlan getJobPlan() {
     return jobPlan;
@@ -508,7 +508,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
       readLock.unlock();
     }
   }
-  
+
   // monitoring apis
   @Override
   public DAGStatusBuilder getDAGStatus() {
@@ -552,11 +552,12 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
     }
     return vertex.getVertexStatus();
   }
-  
-  
+
+
   protected void startRootVertices() {
     for (Vertex v : vertices.values()) {
       if (v.getInputVerticesCount() == 0) {
+        LOG.info("DEBUG: Starting root vertex " + v.getName());
         eventHandler.handle(new VertexEvent(v.getVertexId(),
             VertexEventType.V_START));
       }
@@ -605,7 +606,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
       writeLock.unlock();
     }
   }
-  
+
   @Private
   public DAGState getInternalState() {
     readLock.lock();
@@ -727,7 +728,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   public String getUserName() {
     return userName;
   }
-  
+
   @Override
   public String getQueueName() {
     return queueName;
@@ -862,7 +863,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
 
         dag.edges = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dag.getJobPlan().getEdgeList());
         Map<String,EdgePlan> edgePlans = DagTypeConverters.createEdgePlanMapFromDAGPlan(dag.getJobPlan().getEdgeList());
-        
+
         // setup the dag
         for (Vertex v : dag.vertices.values()) {
           parseVertexEdges(dag, edgePlans, v);
@@ -888,10 +889,10 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
 
     private VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
       TezVertexID vertexId = TezBuilderUtils.newVertexID(dag.getID(), vId);
-      
+
       VertexPlan vertexPlan = dag.getJobPlan().getVertex(vId);
       VertexLocationHint vertexLocationHint = DagTypeConverters.convertFromDAGPlan(vertexPlan.getTaskLocationHintList());
-        
+
       return new VertexImpl(
           vertexId, vertexPlan, vertexName, dag.conf,
           dag.eventHandler, dag.taskAttemptListener,
@@ -912,18 +913,18 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
 
       for(String inEdgeId : vertexPlan.getInEdgeIdList()){
         EdgePlan edgePlan = edgePlans.get(inEdgeId);
-        Vertex inVertex = dag.vertexMap.get(edgePlan.getInputVertexName());    
+        Vertex inVertex = dag.vertexMap.get(edgePlan.getInputVertexName());
         EdgeProperty edgeProp = dag.edges.get(inEdgeId);
         inVertices.put(inVertex, edgeProp);
       }
-      
+
       for(String outEdgeId : vertexPlan.getOutEdgeIdList()){
         EdgePlan edgePlan = edgePlans.get(outEdgeId);
-        Vertex outVertex = dag.vertexMap.get(edgePlan.getOutputVertexName());    
+        Vertex outVertex = dag.vertexMap.get(edgePlan.getOutputVertexName());
         EdgeProperty edgeProp = dag.edges.get(outEdgeId);
         outVertices.put(outVertex, edgeProp);
       }
-      
+
       vertex.setInputVertices(inVertices);
       vertex.setOutputVertices(outVertices);
     }
@@ -931,9 +932,9 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
     protected void setup(DAGImpl job) throws IOException {
       job.initTime = job.clock.getTime();
       String dagIdString = job.dagId.toString();
-      
+
       dagIdString.replace("application", "job");
-      
+
       // TODO remove - TEZ-71
       String user =
         UserGroupInformation.getCurrentUser().getShortUserName();
@@ -1165,7 +1166,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
       }
     }
   }
-  
+
   private static class DAGSchedulerUpdateTransition implements
   SingleArcTransition<DAGImpl, DAGEvent> {
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8298190d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 20d13cd..ce4c609 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -141,7 +141,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   // TODO Metrics
   //private final MRAppMetrics metrics;
   private final AppContext appContext;
-  
+
   private boolean lazyTasksCopyNeeded = false;
   volatile Map<TezTaskID, Task> tasks = new LinkedHashMap<TezTaskID, Task>();
   private Object fullCountersLock = new Object();
@@ -154,7 +154,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
 
   private int numStartedSourceVertices = 0;
   private int distanceFromRoot = 0;
-  
+
   private List<TezDependentTaskCompletionEvent> sourceTaskAttemptCompletionEvents;
   private final List<String> diagnostics = new ArrayList<String>();
 
@@ -167,7 +167,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
 
   List<InputSpec> inputSpecList;
   List<OutputSpec> outputSpecList;
-  
+
   private static final InternalErrorTransition
       INTERNAL_ERROR_TRANSITION = new InternalErrorTransition();
   private static final TaskAttemptCompletedEventTransition
@@ -200,7 +200,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
           .addTransition(VertexState.INITED, VertexState.INITED,
               VertexEventType.V_SOURCE_VERTEX_STARTED,
               new SourceVertexStartedTransition())
-          .addTransition(VertexState.INITED, VertexState.RUNNING, 
+          .addTransition(VertexState.INITED, VertexState.RUNNING,
               VertexEventType.V_START,
               new StartTransition())
 
@@ -220,7 +220,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
               SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
           .addTransition
               (VertexState.RUNNING,
-              EnumSet.of(VertexState.RUNNING, VertexState.KILLED,
+              EnumSet.of(VertexState.RUNNING,
                   VertexState.SUCCEEDED, VertexState.FAILED),
               VertexEventType.V_TASK_COMPLETED,
               new TaskCompletedTransition())
@@ -335,14 +335,14 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   private Token<JobTokenIdentifier> jobToken;
 
   private final TezVertexID vertexId;  //runtime assigned id.
-  private final VertexPlan vertexPlan;  
+  private final VertexPlan vertexPlan;
 
   private final String vertexName;
   private final String processorName;
 
   private Map<Vertex, EdgeProperty> sourceVertices;
   private Map<Vertex, EdgeProperty> targetVertices;
-  
+
   private VertexScheduler vertexScheduler;
 
   private VertexOutputCommitter committer;
@@ -351,8 +351,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   private Map<String, LocalResource> localResources;
   private Map<String, String> environment;
   private final String javaOpts;
-  
-  public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, 
+
+  public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan,
       String vertexName, TezConfiguration conf, EventHandler eventHandler,
       TaskAttemptListener taskAttemptListener,
       Token<JobTokenIdentifier> jobToken,
@@ -384,13 +384,13 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     this.jobToken = jobToken;
     this.committer = new NullVertexOutputCommitter();
     this.vertexLocationHint = vertexLocationHint;
-    
+
     this.taskResource = DagTypeConverters.CreateResourceRequestFromTaskConfig(vertexPlan.getTaskConfig());
-    this.processorName = vertexPlan.hasProcessorName() ? vertexPlan.getProcessorName() : null;  
+    this.processorName = vertexPlan.hasProcessorName() ? vertexPlan.getProcessorName() : null;
     this.localResources = DagTypeConverters.createLocalResourceMapFromDAGPlan(vertexPlan.getTaskConfig().getLocalResourceList());
     this.environment = DagTypeConverters.createEnvironmentMapFromDAGPlan(vertexPlan.getTaskConfig().getEnvironmentSettingList());
     this.javaOpts = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan.getTaskConfig().getJavaOpts() : null;
-    
+
     // This "this leak" is okay because the retained pointer is in an
     //  instance variable.
     stateMachine = stateMachineFactory.make(this);
@@ -404,12 +404,12 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   public TezVertexID getVertexId() {
     return vertexId;
   }
-  
+
   @Override
   public VertexPlan getVertexPlan() {
     return vertexPlan;
   }
-  
+
   @Override
   public int getDistanceFromRoot() {
     return distanceFromRoot;
@@ -521,7 +521,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       this.readLock.unlock();
     }
   }
-  
+
   @Override
   public ProgressBuilder getVertexProgress() {
     this.readLock.lock();
@@ -537,7 +537,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       this.readLock.unlock();
     }
   }
-  
+
   @Override
   public VertexStatusBuilder getVertexStatus() {
     this.readLock.lock();
@@ -631,7 +631,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       writeLock.unlock();
     }
   }
-  
+
   private VertexState getInternalState() {
     readLock.lock();
     try {
@@ -710,7 +710,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       vertex.abortVertex(VertexStatus.State.FAILED);
       return vertex.finished(VertexState.FAILED);
     }
-    
+
     if(vertex.succeededTaskCount == vertex.tasks.size()) {
       try {
         if (!vertex.committed.getAndSet(true)) {
@@ -721,9 +721,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
         LOG.error("Failed to do commit on vertex, name=" + vertex.getName(), e);
         return vertex.finished(VertexState.FAILED);
       }
-      return vertex.finished(VertexState.SUCCEEDED);      
+      return vertex.finished(VertexState.SUCCEEDED);
     }
-    
+
     if (vertex.completedTaskCount == vertex.tasks.size()) {
       // this means the vertex has some killed tasks
       assert vertex.killedTaskCount > 0;
@@ -731,9 +731,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       vertex.abortVertex(VertexStatus.State.KILLED);
       return vertex.finished(VertexState.KILLED);
     }
-    
+
     //return the current state, Vertex not finished yet
-    return vertex.getInternalState();    
+    return vertex.getInternalState();
   }
 
   VertexState finished(VertexState finalState) {
@@ -774,9 +774,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
 
         // TODODAGAM
         // TODO: Splits?
-        
+
         vertex.numTasks = vertex.getVertexPlan().getTaskConfig().getNumTasks();
-        
+
         /*
         TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(job, job.jobId);
         job.numMapTasks = taskSplitMetaInfo.length;
@@ -794,8 +794,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
 
         // create the Tasks but don't start them yet
         createTasks(vertex);
-        
-        
+
+
 
         boolean hasBipartite = false;
         if (vertex.sourceVertices != null) {
@@ -806,10 +806,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
             }
           }
         }
-        
+
         if (hasBipartite) {
           // setup vertex scheduler
-          // TODO this needs to consider data size and perhaps API. 
+          // TODO this needs to consider data size and perhaps API.
           // Currently implicitly BIPARTITE is the only edge type
           vertex.vertexScheduler = new BipartiteSlowStartVertexScheduler(
               vertex,
@@ -907,7 +907,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
 
     @Override
     public void transition(VertexImpl vertex, VertexEvent event) {
-      VertexEventSourceVertexStarted startEvent = 
+      VertexEventSourceVertexStarted startEvent =
                                       (VertexEventSourceVertexStarted) event;
       int distanceFromRoot = startEvent.getSourceDistanceFromRoot() + 1;
       if(vertex.distanceFromRoot < distanceFromRoot) {
@@ -916,8 +916,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       vertex.numStartedSourceVertices++;
       if (vertex.numStartedSourceVertices == vertex.sourceVertices.size()) {
         // Consider inlining this.
-        LOG.info("Starting vertex: " + vertex.getVertexId() + 
-                 " with name: " + vertex.getName() + 
+        LOG.info("Starting vertex: " + vertex.getVertexId() +
+                 " with name: " + vertex.getName() +
                  " with distanceFromRoot: " + vertex.distanceFromRoot );
         vertex.eventHandler.handle(new VertexEvent(vertex.vertexId,
             VertexEventType.V_START));
@@ -1040,10 +1040,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       //eventId is equal to index in the arraylist
       tce.setEventId(vertex.sourceTaskAttemptCompletionEvents.size());
       vertex.sourceTaskAttemptCompletionEvents.add(tce);
-      // TODO this needs to be ordered/grouped by source vertices or else 
-      // my tasks will not know which events are for which vertices' tasks. This 
+      // TODO this needs to be ordered/grouped by source vertices or else
+      // my tasks will not know which events are for which vertices' tasks. This
       // differentiation was not needed for MR because there was only 1 M stage.
-      // if the tce is sent to the task then a solution could be to add vertex 
+      // if the tce is sent to the task then a solution could be to add vertex
       // name to the tce
       // need to send vertex name and task index in that vertex
 
@@ -1060,7 +1060,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
         }
         vertex.successSourceAttemptCompletionEventNoMap.put(taskId, tce.getEventId());
       }
-      
+
       vertex.vertexScheduler.onSourceTaskCompleted(attemptId);
     }
   }
@@ -1073,7 +1073,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       TezDependentTaskCompletionEvent tce =
         ((VertexEventTaskAttemptCompleted) event).getCompletionEvent();
 
-      // TODO this should only be sent for successful events? looks like all 
+      // TODO this should only be sent for successful events? looks like all
       // need to be sent in the existing shuffle code
       // Notify all target vertices
       if (vertex.targetVertices != null) {
@@ -1245,7 +1245,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   public TezDAGID getDAGId() {
     return getDAG().getID();
   }
-  
+
   @Override
   public ApplicationAttemptId getApplicationAttemptId() {
     return appContext.getApplicationAttemptId();
@@ -1259,7 +1259,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   public DAG getDAG() {
     return appContext.getDAG();
   }
-  
+
   @VisibleForTesting
   String getProcessorName() {
     return this.processorName;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8298190d/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
new file mode 100644
index 0000000..3adcd8b
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -0,0 +1,626 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.SystemClock;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.oldrecords.TaskState;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeConnectionPattern;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeSourceType;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskHeartbeatHandler;
+import org.apache.tez.dag.app.dag.DAGState;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.VertexState;
+import org.apache.tez.dag.app.dag.event.DAGEvent;
+import org.apache.tez.dag.app.dag.event.DAGEventType;
+import org.apache.tez.dag.app.dag.event.DAGEventVertexCompleted;
+import org.apache.tez.dag.app.dag.event.DAGFinishEvent;
+import org.apache.tez.dag.app.dag.event.TaskEvent;
+import org.apache.tez.dag.app.dag.event.TaskEventType;
+import org.apache.tez.dag.app.dag.event.VertexEvent;
+import org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted;
+import org.apache.tez.dag.app.dag.event.VertexEventType;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.avro.HistoryEventType;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.engine.common.security.JobTokenSecretManager;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestDAGImpl {
+
+  private static final Log LOG = LogFactory.getLog(TestDAGImpl.class);
+  private DAGPlan dagPlan;
+  private TezDAGID dagId;
+  private TezConfiguration conf;
+  private DrainDispatcher dispatcher;
+  private Credentials fsTokens;
+  private AppContext appContext;
+  private ApplicationAttemptId appAttemptId;
+  private DAGImpl dag;
+  private VertexEventDispatcher vertexEventDispatcher;
+  private DagEventDispatcher dagEventDispatcher;
+  private TaskAttemptListener taskAttemptListener;
+  private TaskHeartbeatHandler thh;
+  private Clock clock = new SystemClock();
+  private JobTokenSecretManager jobTokenSecretManager;
+  private DAGFinishEventHandler dagFinishEventHandler;
+
+  private class DagEventDispatcher implements EventHandler<DAGEvent> {
+    @Override
+    public void handle(DAGEvent event) {
+      dag.handle(event);
+    }
+  }
+
+  private class HistoryHandler implements EventHandler<DAGHistoryEvent> {
+    @Override
+    public void handle(DAGHistoryEvent event) {
+    }
+  }
+
+  private class TaskEventHandler implements EventHandler<TaskEvent> {
+    @Override
+    public void handle(TaskEvent event) {
+    }
+  }
+
+  private class VertexEventDispatcher
+      implements EventHandler<VertexEvent> {
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public void handle(VertexEvent event) {
+      Vertex vertex = dag.getVertex(event.getVertexId());
+      ((EventHandler<VertexEvent>) vertex).handle(event);
+    }
+  }
+
+  private class DAGFinishEventHandler
+  implements EventHandler<DAGFinishEvent> {
+    public int dagFinishEvents = 0;
+
+    @Override
+    public void handle(DAGFinishEvent event) {
+      ++dagFinishEvents;
+    }
+  }
+
+  private DAGPlan createTestDAGPlan() {
+    LOG.info("Setting up dag plan");
+    DAGPlan dag = DAGPlan.newBuilder()
+        .setName("testverteximpl")
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex1")
+            .setType(PlanVertexType.NORMAL)
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host1")
+                .addRack("rack1")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(1)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("")
+                .setTaskModule("x1.y1")
+                .build()
+                )
+            .addOutEdgeId("e1")
+            .build()
+            )
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex2")
+            .setType(PlanVertexType.NORMAL)
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host2")
+                .addRack("rack2")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(2)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("")
+                .setTaskModule("x2.y2")
+                .build()
+                )
+            .addOutEdgeId("e2")
+            .build()
+            )
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex3")
+            .setType(PlanVertexType.NORMAL)
+            .setProcessorName("x3.y3")
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host3")
+                .addRack("rack3")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(2)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("foo")
+                .setTaskModule("x3.y3")
+                .build()
+                )
+            .addInEdgeId("e1")
+            .addInEdgeId("e2")
+            .addOutEdgeId("e3")
+            .addOutEdgeId("e4")
+            .build()
+            )
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex4")
+            .setType(PlanVertexType.NORMAL)
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host4")
+                .addRack("rack4")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(2)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("")
+                .setTaskModule("x4.y4")
+                .build()
+                )
+            .addInEdgeId("e3")
+            .addOutEdgeId("e5")
+            .build()
+            )
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex5")
+            .setType(PlanVertexType.NORMAL)
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host5")
+                .addRack("rack5")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(2)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("")
+                .setTaskModule("x5.y5")
+                .build()
+                )
+            .addInEdgeId("e4")
+            .addOutEdgeId("e6")
+            .build()
+            )
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex6")
+            .setType(PlanVertexType.NORMAL)
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host6")
+                .addRack("rack6")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(2)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("")
+                .setTaskModule("x6.y6")
+                .build()
+                )
+            .addInEdgeId("e5")
+            .addInEdgeId("e6")
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i3_v1")
+            .setInputVertexName("vertex1")
+            .setOutputClass("o1")
+            .setOutputVertexName("vertex3")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e1")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i3_v2")
+            .setInputVertexName("vertex2")
+            .setOutputClass("o2")
+            .setOutputVertexName("vertex3")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e2")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i4_v3")
+            .setInputVertexName("vertex3")
+            .setOutputClass("o3_v4")
+            .setOutputVertexName("vertex4")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e3")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i5_v3")
+            .setInputVertexName("vertex3")
+            .setOutputClass("o3_v5")
+            .setOutputVertexName("vertex5")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e4")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i6_v4")
+            .setInputVertexName("vertex4")
+            .setOutputClass("o4")
+            .setOutputVertexName("vertex6")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e5")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i6_v5")
+            .setInputVertexName("vertex5")
+            .setOutputClass("o5")
+            .setOutputVertexName("vertex6")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e6")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .build();
+
+    return dag;
+  }
+
+  @Before
+  public void setup() {
+    conf = new TezConfiguration();
+    appAttemptId = BuilderUtils.newApplicationAttemptId(
+        BuilderUtils.newApplicationId(100, 1), 1);
+    dagId = new TezDAGID(appAttemptId.getApplicationId(), 1);
+    Assert.assertNotNull(dagId);
+    dagPlan = createTestDAGPlan();
+    dispatcher = new DrainDispatcher();
+    fsTokens = new Credentials();
+    jobTokenSecretManager = new JobTokenSecretManager();
+    appContext = mock(AppContext.class);
+    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
+    doReturn(dagId).when(appContext).getDAGID();
+    dag = new DAGImpl(dagId, appAttemptId, conf, dagPlan,
+        dispatcher.getEventHandler(),  taskAttemptListener,
+        jobTokenSecretManager, fsTokens, clock, "user", 10000, thh, appContext);
+    doReturn(dag).when(appContext).getDAG();
+    vertexEventDispatcher = new VertexEventDispatcher();
+    dispatcher.register(VertexEventType.class, vertexEventDispatcher);
+    dagEventDispatcher = new DagEventDispatcher();
+    dispatcher.register(DAGEventType.class, dagEventDispatcher);
+    dispatcher.register(HistoryEventType.class,
+        new HistoryHandler());
+    dagFinishEventHandler = new DAGFinishEventHandler();
+    dispatcher.register(DAGFinishEvent.Type.class, dagFinishEventHandler);
+    dispatcher.register(TaskEventType.class, new TaskEventHandler());
+    dispatcher.init(conf);
+    dispatcher.start();
+  }
+
+  @After
+  public void teardown() {
+    dagPlan = null;
+    dag = null;
+    dispatcher.await();
+    dispatcher.stop();
+  }
+
+  private void initDAG(DAGImpl dag) {
+    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_INIT));
+    Assert.assertEquals(DAGState.INITED, dag.getState());
+  }
+
+  private void startDAG(DAGImpl dag) {
+    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_START));
+    Assert.assertEquals(DAGState.RUNNING, dag.getState());
+  }
+
+  @Test
+  public void testDAGInit() {
+    initDAG(dag);
+    Assert.assertEquals(6, dag.getTotalVertices());
+  }
+
+  @Test
+  public void testDAGStart() {
+    initDAG(dag);
+    startDAG(dag);
+    dispatcher.await();
+
+    for (int i = 0 ; i < 6; ++i ) {
+      TezVertexID vId = new TezVertexID(dagId, i);
+      Vertex v = dag.getVertex(vId);
+      Assert.assertEquals(VertexState.RUNNING, v.getState());
+      if (i < 2) {
+        Assert.assertEquals(0, v.getDistanceFromRoot());
+      } else if (i == 2) {
+        Assert.assertEquals(1, v.getDistanceFromRoot());
+      } else if ( i > 2 && i < 5) {
+        Assert.assertEquals(2, v.getDistanceFromRoot());
+      } else if (i == 5) {
+        Assert.assertEquals(3, v.getDistanceFromRoot());
+      }
+    }
+
+    for (int i = 0 ; i < 6; ++i ) {
+      TezVertexID vId = new TezVertexID(dagId, i);
+      LOG.info("Distance from root: v" + i + ":"
+          + dag.getVertex(vId).getDistanceFromRoot());
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  @Test
+  public void testVertexCompletion() {
+    initDAG(dag);
+    startDAG(dag);
+    dispatcher.await();
+
+    TezVertexID vId = new TezVertexID(dagId, 1);
+    Vertex v = dag.getVertex(vId);
+    ((EventHandler<VertexEvent>) v).handle(new VertexEventTaskCompleted(
+        new TezTaskID(vId, 0), TaskState.SUCCEEDED));
+    ((EventHandler<VertexEvent>) v).handle(new VertexEventTaskCompleted(
+        new TezTaskID(vId, 1), TaskState.SUCCEEDED));
+    dispatcher.await();
+
+    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+    Assert.assertEquals(1, dag.getSuccessfulVertices());
+  }
+
+  public void testKillStartedDAG() {
+    initDAG(dag);
+    startDAG(dag);
+    dispatcher.await();
+
+    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+    dispatcher.await();
+
+    Assert.assertEquals(DAGState.KILLED, dag.getState());
+    for (int i = 0 ; i < 6; ++i ) {
+      TezVertexID vId = new TezVertexID(dagId, i);
+      Vertex v = dag.getVertex(vId);
+      Assert.assertEquals(VertexState.KILLED, v.getState());
+    }
+
+  }
+
+  @SuppressWarnings("unchecked")
+  @Test
+  public void testKillRunningDAG() {
+    initDAG(dag);
+    startDAG(dag);
+    dispatcher.await();
+
+    TezVertexID vId1 = new TezVertexID(dagId, 1);
+    Vertex v1 = dag.getVertex(vId1);
+    ((EventHandler<VertexEvent>) v1).handle(new VertexEventTaskCompleted(
+        new TezTaskID(vId1, 0), TaskState.SUCCEEDED));
+    TezVertexID vId0 = new TezVertexID(dagId, 0);
+    Vertex v0 = dag.getVertex(vId0);
+    ((EventHandler<VertexEvent>) v0).handle(new VertexEventTaskCompleted(
+        new TezTaskID(vId0, 0), TaskState.SUCCEEDED));
+    dispatcher.await();
+
+    Assert.assertEquals(VertexState.SUCCEEDED, v0.getState());
+    Assert.assertEquals(VertexState.RUNNING, v1.getState());
+
+    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+    dispatcher.await();
+
+    Assert.assertEquals(DAGState.KILL_WAIT, dag.getState());
+    Assert.assertEquals(VertexState.SUCCEEDED, v0.getState());
+    Assert.assertEquals(VertexState.KILL_WAIT, v1.getState());
+    for (int i = 2 ; i < 6; ++i ) {
+      TezVertexID vId = new TezVertexID(dagId, i);
+      Vertex v = dag.getVertex(vId);
+      Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+    }
+    Assert.assertEquals(1, dag.getSuccessfulVertices());
+  }
+
+  @Test
+  public void testInvalidEvent() {
+    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_START));
+    dispatcher.await();
+    Assert.assertEquals(DAGState.ERROR, dag.getState());
+  }
+
+  @Test
+  @Ignore
+  public void testVertexSuccessfulCompletionUpdates() {
+    initDAG(dag);
+    startDAG(dag);
+    dispatcher.await();
+
+    for (int i = 0; i < 6; ++i) {
+      dag.handle(new DAGEventVertexCompleted(
+          new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
+    }
+    dispatcher.await();
+    Assert.assertEquals(DAGState.RUNNING, dag.getState());
+    Assert.assertEquals(1, dag.getSuccessfulVertices());
+
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 2), VertexState.SUCCEEDED));
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 3), VertexState.SUCCEEDED));
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 4), VertexState.SUCCEEDED));
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 5), VertexState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(DAGState.SUCCEEDED, dag.getState());
+    Assert.assertEquals(6, dag.getSuccessfulVertices());
+  }
+
+  @Test
+  @Ignore
+  public void testVertexFailureHandling() {
+    initDAG(dag);
+    startDAG(dag);
+    dispatcher.await();
+
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 2), VertexState.FAILED));
+    dispatcher.await();
+    Assert.assertEquals(DAGState.FAILED, dag.getState());
+    Assert.assertEquals(2, dag.getSuccessfulVertices());
+
+    // Expect running vertices to be killed on first failure
+    for (int i = 3; i < 6; ++i) {
+      TezVertexID vId = new TezVertexID(dagId, i);
+      Vertex v = dag.getVertex(vId);
+      Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+    }
+  }
+
+  @Test
+  @Ignore
+  public void testDAGKill() {
+    initDAG(dag);
+    startDAG(dag);
+    dispatcher.await();
+
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
+    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+
+    for (int i = 2; i < 6; ++i) {
+      dag.handle(new DAGEventVertexCompleted(
+          new TezVertexID(dagId, i), VertexState.SUCCEEDED));
+    }
+    dispatcher.await();
+    Assert.assertEquals(DAGState.KILLED, dag.getState());
+    Assert.assertEquals(6, dag.getSuccessfulVertices());
+    Assert.assertEquals(1, dagFinishEventHandler.dagFinishEvents);
+  }
+
+  @Test
+  public void testDAGKillPending() {
+    initDAG(dag);
+    startDAG(dag);
+    dispatcher.await();
+
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
+    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+
+    for (int i = 2; i < 5; ++i) {
+      dag.handle(new DAGEventVertexCompleted(
+          new TezVertexID(dagId, i), VertexState.SUCCEEDED));
+    }
+    dispatcher.await();
+    Assert.assertEquals(DAGState.KILL_WAIT, dag.getState());
+
+    dag.handle(new DAGEventVertexCompleted(
+        new TezVertexID(dagId, 5), VertexState.KILLED));
+    dispatcher.await();
+    Assert.assertEquals(DAGState.KILLED, dag.getState());
+    Assert.assertEquals(5, dag.getSuccessfulVertices());
+    Assert.assertEquals(1, dagFinishEventHandler.dagFinishEvents);
+  }
+
+  @Test
+  public void testDiagnosticUpdates() {
+    // FIXME need to implement
+  }
+
+  @Test
+  public void testCounterUpdates() {
+    // FIXME need to implement
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8298190d/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index fe6ae14..f78a7b8 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -62,6 +62,8 @@ import org.apache.tez.dag.app.dag.Vertex;
 import org.apache.tez.dag.app.dag.VertexState;
 import org.apache.tez.dag.app.dag.event.DAGEvent;
 import org.apache.tez.dag.app.dag.event.DAGEventType;
+import org.apache.tez.dag.app.dag.event.TaskEvent;
+import org.apache.tez.dag.app.dag.event.TaskEventType;
 import org.apache.tez.dag.app.dag.event.VertexEvent;
 import org.apache.tez.dag.app.dag.event.VertexEventTaskAttemptCompleted;
 import org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted;
@@ -115,7 +117,7 @@ public class TestVertexImpl {
     public int abortCounter = 0;
     private boolean throwError;
     private boolean throwErrorOnAbort;
-    
+
     public CountingVertexOutputCommitter(boolean throwError,
         boolean throwOnAbort) {
       this.throwError = throwError;
@@ -125,7 +127,7 @@ public class TestVertexImpl {
     public CountingVertexOutputCommitter() {
       this(false, false);
     }
-    
+
     @Override
     public void init(VertexContext context) throws IOException {
       ++initCounter;
@@ -150,12 +152,26 @@ public class TestVertexImpl {
       if (throwErrorOnAbort) {
         throw new IOException("I can throwz exceptions in abort");
       }
-    }    
+    }
   }
-  
+
+  private class TaskEventHandler implements EventHandler<TaskEvent> {
+    @Override
+    public void handle(TaskEvent event) {
+    }
+  }
+
   private class DagEventDispatcher implements EventHandler<DAGEvent> {
+    public Map<DAGEventType, Integer> eventCount =
+        new HashMap<DAGEventType, Integer>();
+
     @Override
     public void handle(DAGEvent event) {
+      int count = 1;
+      if (eventCount.containsKey(event.getType())) {
+        count = eventCount.get(event.getType()) + 1;
+      }
+      eventCount.put(event.getType(), count);
     }
   }
 
@@ -164,7 +180,7 @@ public class TestVertexImpl {
     public void handle(DAGHistoryEvent event) {
     }
   }
-  
+
   private class VertexEventDispatcher
       implements EventHandler<VertexEvent> {
 
@@ -421,7 +437,7 @@ public class TestVertexImpl {
 
       Map<Vertex, EdgeProperty> outVertices =
           new HashMap<Vertex, EdgeProperty>();
-      
+
       for(String inEdgeId : vertexPlan.getInEdgeIdList()){
         EdgePlan edgePlan = edgePlans.get(inEdgeId);
         Vertex inVertex = this.vertices.get(edgePlan.getInputVertexName());
@@ -469,6 +485,7 @@ public class TestVertexImpl {
     dispatcher.register(DAGEventType.class, dagEventDispatcher);
     dispatcher.register(HistoryEventType.class,
         new HistoryHandler());
+    dispatcher.register(TaskEventType.class, new TaskEventHandler());
     dispatcher.init(conf);
     dispatcher.start();
   }
@@ -500,10 +517,10 @@ public class TestVertexImpl {
     if (checkKillWait) {
       Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
     } else {
-      Assert.assertEquals(VertexState.KILLED, v.getState());      
+      Assert.assertEquals(VertexState.KILLED, v.getState());
     }
   }
-  
+
   private void startVertex(VertexImpl v,
       boolean checkRunningState) {
     Assert.assertEquals(VertexState.INITED, v.getState());
@@ -522,10 +539,10 @@ public class TestVertexImpl {
 
     VertexImpl v3 = vertices.get("vertex3");
     initVertex(v3);
-    
+
     Assert.assertEquals("x3.y3", v3.getProcessorName());
     Assert.assertEquals("foo", v3.getJavaOpts());
-    
+
     Assert.assertEquals(2, v3.getInputSpecList().size());
     Assert.assertEquals(2, v3.getInputVerticesCount());
     Assert.assertEquals(2, v3.getOutputVerticesCount());
@@ -547,7 +564,7 @@ public class TestVertexImpl {
         .getInputClassName())
         || "i3_v2".equals(v3.getInputSpecList().get(1)
             .getInputClassName()));
-    
+
     Assert.assertTrue("vertex4".equals(v3.getOutputSpecList().get(0)
         .getVertexName())
         || "vertex5".equals(v3.getOutputSpecList().get(0)
@@ -606,7 +623,7 @@ public class TestVertexImpl {
     v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.RUNNING, v.getState());
-    
+
     v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.RUNNING, v.getState());
@@ -632,7 +649,7 @@ public class TestVertexImpl {
         StringUtils.join(",", v.getDiagnostics()).toLowerCase();
     Assert.assertTrue(diagnostics.contains("task failed " + t1.toString()));
   }
-  
+
   @Test
   public void testVertexWithNoTasks() {
     // FIXME a vertex with no tasks should not be allowed
@@ -644,14 +661,14 @@ public class TestVertexImpl {
   }
 
   @Test
-  public void testVertexKill() {
+  public void testVertexKillDiagnostics() {
     VertexImpl v1 = vertices.get("vertex1");
     killVertex(v1, false);
     String diagnostics =
         StringUtils.join(",", v1.getDiagnostics()).toLowerCase();
     Assert.assertTrue(diagnostics.contains(
         "vertex received kill in new state"));
-    
+
     VertexImpl v2 = vertices.get("vertex2");
     initVertex(v2);
     killVertex(v2, false);
@@ -672,30 +689,73 @@ public class TestVertexImpl {
   }
 
   @Test
-  public void testKilledTasksHandling() {
+  public void testVertexKillPending() {
     VertexImpl v = vertices.get("vertex2");
     initVertex(v);
+    VertexImpl v3 = vertices.get("vertex3");
+    initVertex(v3);
+
     startVertex(v);
 
-    TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
-    TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+    v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+    Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+
+    v.handle(new VertexEventTaskCompleted(
+        new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
+    Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
 
-    v.handle(new VertexEventTaskCompleted(t1, TaskState.KILLED));
+    v.handle(new VertexEventTaskCompleted(
+        new TezTaskID(v.getVertexId(), 1), TaskState.KILLED));
     dispatcher.await();
-    Assert.assertEquals(VertexState.RUNNING, v.getState());
+    Assert.assertEquals(VertexState.KILLED, v.getState());
+  }
+
+  @Test
+  @Ignore
+  public void testVertexKill() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    VertexImpl v3 = vertices.get("vertex3");
+    initVertex(v3);
 
-    v.handle(new VertexEventTaskCompleted(t2, TaskState.KILLED));
+    startVertex(v);
+
+    v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+    Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+
+    v.handle(new VertexEventTaskCompleted(
+        new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
+    Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+
+    v.handle(new VertexEventTaskCompleted(
+        new TezTaskID(v.getVertexId(), 1), TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.KILLED, v.getState());
   }
 
   @Test
+  @Ignore
+  public void testKilledTasksHandling() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    startVertex(v);
+
+    TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+    TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.FAILED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.FAILED, v.getState());
+    Assert.assertEquals(TaskState.KILLED, v.getTask(t2).getState());
+  }
+
+  @Test
   public void testVertexCommitterInit() {
     VertexImpl v2 = vertices.get("vertex2");
     initVertex(v2);
     Assert.assertTrue(v2.getVertexOutputCommitter()
         instanceof NullVertexOutputCommitter);
-    
+
     VertexImpl v6 = vertices.get("vertex6");
     initVertex(v6);
     Assert.assertTrue(v6.getVertexOutputCommitter()
@@ -708,13 +768,13 @@ public class TestVertexImpl {
     initVertex(v2);
     Assert.assertTrue(v2.getVertexScheduler()
         instanceof ImmediateStartVertexScheduler);
-    
+
     VertexImpl v6 = vertices.get("vertex6");
     initVertex(v6);
     Assert.assertTrue(v6.getVertexScheduler()
         instanceof BipartiteSlowStartVertexScheduler);
   }
-  
+
   @Test
   public void testVertexTaskFailure() {
     VertexImpl v = vertices.get("vertex2");
@@ -730,7 +790,7 @@ public class TestVertexImpl {
     v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.RUNNING, v.getState());
-    
+
     v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
     v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
     dispatcher.await();
@@ -765,9 +825,9 @@ public class TestVertexImpl {
   public void testDiagnostics() {
     // FIXME need to test diagnostics in various cases
   }
-  
+
   @Test
-  public void testTaskAttemptCompletionEvents() {    
+  public void testTaskAttemptCompletionEvents() {
     // FIXME need to test handling of task attempt events
   }
 
@@ -821,7 +881,7 @@ public class TestVertexImpl {
     v4.handle(new VertexEventTaskAttemptCompleted(cEvt3));
     v5.handle(new VertexEventTaskAttemptCompleted(cEvt4));
     v5.handle(new VertexEventTaskAttemptCompleted(cEvt5));
-    v5.handle(new VertexEventTaskAttemptCompleted(cEvt6));    
+    v5.handle(new VertexEventTaskAttemptCompleted(cEvt6));
 
     v4.handle(new VertexEventTaskCompleted(t1_v4, TaskState.SUCCEEDED));
     v4.handle(new VertexEventTaskCompleted(t2_v4, TaskState.SUCCEEDED));
@@ -850,7 +910,9 @@ public class TestVertexImpl {
     v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
-
+    Assert.assertEquals(1,
+        dagEventDispatcher.eventCount.get(
+            DAGEventType.DAG_VERTEX_COMPLETED).intValue());
   }
 
   @Test
@@ -861,7 +923,7 @@ public class TestVertexImpl {
     CountingVertexOutputCommitter committer =
         new CountingVertexOutputCommitter();
     v.setVertexOutputCommitter(committer);
-    
+
     startVertex(v);
 
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
@@ -873,14 +935,14 @@ public class TestVertexImpl {
     // v.handle(new VertexEventTaskReschedule(t1));
     v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
-    Assert.assertEquals(VertexState.RUNNING, v.getState());    
+    Assert.assertEquals(VertexState.RUNNING, v.getState());
     Assert.assertEquals(0, committer.commitCounter);
-    
+
     v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     dispatcher.await();
-    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());    
+    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
     Assert.assertEquals(1, committer.commitCounter);
-    
+
   }
 
   @Test
@@ -890,7 +952,7 @@ public class TestVertexImpl {
     CountingVertexOutputCommitter committer =
         new CountingVertexOutputCommitter();
     v.setVertexOutputCommitter(committer);
-    
+
     startVertex(v);
 
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
@@ -900,23 +962,23 @@ public class TestVertexImpl {
     v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
-    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());    
+    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
     Assert.assertEquals(1, committer.commitCounter);
-    
+
     v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
-    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());    
+    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
     Assert.assertEquals(1, committer.commitCounter);
     Assert.assertEquals(0, committer.abortCounter);
     Assert.assertEquals(0, committer.initCounter); // already done in init
     Assert.assertEquals(0, committer.setupCounter); // already done in init
   }
-  
+
   @Test
   public void testCommitterInitAndSetup() {
     // FIXME need to add a test for this
   }
-  
+
   @Test
   public void testTaskAttemptFetchFailureHandling() {
     // FIXME needs testing
@@ -929,7 +991,7 @@ public class TestVertexImpl {
     CountingVertexOutputCommitter committer =
         new CountingVertexOutputCommitter(true, true);
     v.setVertexOutputCommitter(committer);
-    
+
     startVertex(v);
 
     TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
@@ -938,13 +1000,28 @@ public class TestVertexImpl {
     v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
-    Assert.assertEquals(VertexState.FAILED, v.getState());    
+    Assert.assertEquals(VertexState.FAILED, v.getState());
     Assert.assertEquals(1, committer.commitCounter);
-    
+
     // FIXME need to verify whether abort needs to be called if commit fails
     Assert.assertEquals(0, committer.abortCounter);
     Assert.assertEquals(0, committer.initCounter); // already done in init
-    Assert.assertEquals(0, committer.setupCounter); // already done in init  
+    Assert.assertEquals(0, committer.setupCounter); // already done in init
+  }
+
+  @Test
+  public void testHistoryEventGeneration() {
+  }
+
+  @Test
+  public void testInvalidEvent() {
+    VertexImpl v = vertices.get("vertex2");
+    v.handle(new VertexEvent(v.getVertexId(),
+        VertexEventType.V_START));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.ERROR, v.getState());
+    Assert.assertEquals(1,
+        dagEventDispatcher.eventCount.get(
+            DAGEventType.INTERNAL_ERROR).intValue());
   }
-  
 }


[23/50] [abbrv] git commit: TEZ-152. Add a way for MR tasks to figure out which stage they belong to. (sseth)

Posted by ss...@apache.org.
TEZ-152. Add a way for MR tasks to figure out which stage they belong
to. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/7d401d2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/7d401d2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/7d401d2c

Branch: refs/heads/master
Commit: 7d401d2cd0af2c68ff2a060849b71e1b0796c27d
Parents: ef37823
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue May 21 18:51:32 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue May 21 18:51:32 2013 -0700

----------------------------------------------------------------------
 .../apache/tez/mapreduce/hadoop/MRJobConfig.java   |    2 +
 .../mapreduce/hadoop/MultiStageMRConfigUtil.java   |   17 +++++++++++++++
 .../apache/tez/mapreduce/task/MRRuntimeTask.java   |    1 +
 3 files changed, 20 insertions(+), 0 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7d401d2c/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
index 7e650b1..a42597e 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
@@ -649,6 +649,8 @@ public interface MRJobConfig {
   public static final String MRR_VERTEX_PREFIX = 
       "mrr.vertex.";
 
+  public static final String VERTEX_NAME = "mapreduce.task.vertex.name";
+  
   // Stage specific properties
   // Format of each property is mapred.ireducer.stage.<stage-num>.<suffix>
   // where suffix is one of MRR_INTERMEDIATE_STAGE_* fields defined below.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7d401d2c/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java
index b4b34e0..fd3dd78 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java
@@ -96,14 +96,31 @@ public class MultiStageMRConfigUtil {
   public static String getInitialMapVertexName() {
     return INITIAL_MAP_VERTEX_NAME;
   }
+  
+  public boolean isInitialMapVertex(String vertexName) {
+    return vertexName.equals(INITIAL_MAP_VERTEX_NAME);
+  }
 
   public static String getFinalReduceVertexName() {
     return FINAL_REDUCE_VERTEX_NAME;
   }
 
+  public boolean isFinalReduceVertex(String vertexName) {
+    return vertexName.equals(FINAL_REDUCE_VERTEX_NAME);
+  }
+
   public static String getIntermediateStageVertexName(int stageNum) {
     return INTERMEDIATE_TASK_VERTEX_NAME_PREFIX + stageNum;
   }
+  
+  public static int getIntermediateStageNum(String vertexName) {
+    if (vertexName.matches(INTERMEDIATE_TASK_VERTEX_NAME_PREFIX + "\\d+")) {
+      return Integer.parseInt(vertexName
+          .substring(INTERMEDIATE_TASK_VERTEX_NAME_PREFIX.length()));
+    } else {
+      return -1;
+    }
+  }
 
   // Returns config settings specific to named vertex
   public static Configuration getBasicConfForVertex(Configuration baseConf,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7d401d2c/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index e5dfd66..34aaa1b 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -97,6 +97,7 @@ public class MRRuntimeTask extends RuntimeTask {
     // submit configuration parameters to the AM and effectively tasks via RPC.
 
     final JobConf job = new JobConf(taskConf);
+    job.set(MRJobConfig.VERTEX_NAME, taskContext.getVertexName());
 
     MRTask mrTask = (MRTask) getProcessor();
     this.mrTask = mrTask;


[32/50] [abbrv] git commit: TEZ-27. Create tests for DAG Scheduler (bikas)

Posted by ss...@apache.org.
TEZ-27. Create tests for DAG Scheduler (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/82ee45e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/82ee45e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/82ee45e8

Branch: refs/heads/master
Commit: 82ee45e8d3d7f0168b49c534384217305ab81a6d
Parents: 3bb954b
Author: Bikas Saha <bi...@apache.org>
Authored: Tue May 28 19:20:30 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue May 28 19:20:30 2013 -0700

----------------------------------------------------------------------
 .../org/apache/tez/dag/app/dag/TaskAttempt.java    |    5 +
 .../dag/app/dag/event/DAGEventSchedulerUpdate.java |    2 +-
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |   30 ++-
 .../tez/dag/app/dag/impl/DAGSchedulerMRR.java      |   87 +++----
 .../dag/app/dag/impl/DAGSchedulerNaturalOrder.java |    2 +-
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java      |   18 ++-
 .../tez/dag/app/rm/TaskSchedulerEventHandler.java  |    2 +-
 .../tez/dag/app/dag/impl/TestDAGScheduler.java     |  188 +++++++++++++++
 8 files changed, 277 insertions(+), 57 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
index 712f968..2628f0c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
@@ -27,13 +27,18 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.oldrecords.TaskAttemptReport;
 import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
+import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezVertexID;
 
 /**
  * Read only view of TaskAttempt.
  */
 public interface TaskAttempt {
   TezTaskAttemptID getID();
+  TezVertexID getVertexID();
+  TezDAGID getDAGID();
+  
   TaskAttemptReport getReport();
   List<String> getDiagnostics();
   TezCounters getCounters();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java
index ff11e99..99b872e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java
@@ -30,7 +30,7 @@ public class DAGEventSchedulerUpdate extends DAGEvent {
   private final UpdateType updateType;
   
   public DAGEventSchedulerUpdate(UpdateType updateType, TaskAttempt attempt) {
-    super(attempt.getID().getTaskID().getVertexID().getDAGId(), 
+    super(attempt.getDAGID(), 
           DAGEventType.DAG_SCHEDULER_UPDATE);
     this.attempt = attempt;
     this.updateType = updateType;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index cc6699d..3681c85 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -869,9 +869,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
         for (Vertex v : dag.vertices.values()) {
           parseVertexEdges(dag, edgePlans, v);
         }
-
-        dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
-        //dag.dagScheduler = new DAGSchedulerMRR(dag, dag.eventHandler);
+        
+        assignDAGScheduler(dag);
 
         // TODO Metrics
         //dag.metrics.endPreparingJob(dag);
@@ -887,6 +886,31 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
         return dag.finished(DAGState.FAILED);
       }
     }
+    
+    private void assignDAGScheduler(DAGImpl dag) {
+      boolean isMRR = true;
+      for(Vertex vertex : dag.vertices.values()) {
+        Map<Vertex, EdgeProperty> outVertices = vertex.getOutputVertices();
+        if(outVertices == null || outVertices.isEmpty()) {
+          continue;
+        }
+        if(outVertices.size() > 1 || 
+           outVertices.values().iterator().next().getConnectionPattern() != 
+           EdgeProperty.ConnectionPattern.BIPARTITE) {
+          // more than 1 output OR single output is not bipartite
+          isMRR = false;
+          break;
+        }          
+      }
+      
+      if(isMRR) {
+        LOG.info("Using MRR dag scheduler");
+        dag.dagScheduler = new DAGSchedulerMRR(dag, dag.eventHandler);
+      } else {
+        LOG.info("Using Natural order dag scheduler");
+        dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
+      }
+    }
 
     private VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
       TezVertexID vertexId = TezBuilderUtils.newVertexID(dag.getID(), vId);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
index 12fdb10..d404155 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
@@ -48,7 +48,7 @@ public class DAGSchedulerMRR implements DAGScheduler {
   
   @Override
   public void vertexCompleted(Vertex vertex) {
-    if(currentPartitioner!= null) {
+    if(currentPartitioner != null) {
       if(vertex != currentPartitioner) {
         String message = vertex.getVertexId() + " finished. Expecting "
             + currentPartitioner + " to finish.";
@@ -57,68 +57,59 @@ public class DAGSchedulerMRR implements DAGScheduler {
       }
       LOG.info("Current partitioner " + currentPartitioner.getVertexId()
           + " is completed. " 
-          + (currentShuffler!=null?currentShuffler.getVertexId():"null")
-          + " is new partitioner");
+          + (currentShuffler!=null ? 
+             currentShuffler.getVertexId() + " is new partitioner":
+             "No current shuffler to replace the partitioner"));
       currentPartitioner = currentShuffler;
-      currentShuffler = null;
-    } else {
-      if(vertex != currentShuffler) {
-        String message = vertex.getVertexId() + " finished. Expecting "
-            + currentShuffler.getVertexId() + " to finish";
-        LOG.fatal(message);
-        throw new TezException(message);
-      }      
+      currentShuffler = null;     
     }
   }
   
   @Override
   public void scheduleTask(DAGEventSchedulerUpdate event) {
     TaskAttempt attempt = event.getAttempt();
-    Vertex vertex = dag.getVertex(attempt.getID().getTaskID().getVertexID());
+    Vertex vertex = dag.getVertex(attempt.getVertexID());
     int vertexDistanceFromRoot = vertex.getDistanceFromRoot();
-    if(vertexDistanceFromRoot == 0) {
+    boolean reOrderPriority = false;
+    
+    if(currentPartitioner == null) {
+      // no partitioner. so set it.
       currentPartitioner = vertex;
-      LOG.info(vertex.getVertexId() + " is first partitioner");
+      currentShufflerDepth = vertexDistanceFromRoot;
+      LOG.info(vertex.getVertexId() + " is new partitioner at depth "
+          + vertexDistanceFromRoot);
+    } else if (currentShuffler == null && 
+        vertexDistanceFromRoot > currentShufflerDepth) {
+      // vertex not a partitioner. no shuffler set. has more depth than current
+      // shuffler. this must be the new shuffler.
+      currentShuffler = vertex;
+      currentShufflerDepth = vertexDistanceFromRoot;
+      LOG.info(vertex.getVertexId() + " is new shuffler at depth "
+          + currentShufflerDepth);
     }
-    if(vertexDistanceFromRoot > currentShufflerDepth) {
-      if(currentShuffler == null) {
-        currentShuffler = vertex;
-        currentShufflerDepth = vertexDistanceFromRoot;
-        LOG.info(currentShuffler.getVertexId() + " is new shuffler at depth " + 
-                 currentShufflerDepth);
-      } else {
-        if(currentShufflerDepth+1 == vertexDistanceFromRoot && 
-           currentPartitioner == null
-           ) {
-          currentPartitioner = currentShuffler;
-          currentShuffler = vertex;
-          currentShufflerDepth = vertexDistanceFromRoot;
-          LOG.info("Shuffler " + currentPartitioner.getVertexId() + 
-                   " becomes partitioner as new shuffler " + 
-                   currentShuffler.getVertexId() + " has started at depth " + 
-                   currentShufflerDepth);          
-        } else {
-          String message = vertex.getVertexId()
-              + " has scheduled tasks at depth " + vertexDistanceFromRoot
-              + " greater than depth " + currentShufflerDepth
-              + " of current shuffler " + currentShuffler.getVertexId()
-              + ". Unexpected.";
-          LOG.fatal(message);
-          throw new TezException(message);
-        }
-      }
+    
+    if(currentShuffler == vertex) {
+      // current shuffler vertex. assign special priority
+      reOrderPriority = true;
     }
+    
+    // sanity check
+    if(currentPartitioner != vertex && currentShuffler != vertex) {
+      String message = vertex.getVertexId() + " is neither the "
+          + " current partitioner: " + currentPartitioner.getVertexId()
+          + " nor the current shuffler: " + currentShuffler.getVertexId();
+      LOG.fatal(message);
+      throw new TezException(message);      
+    }    
 
     // natural priority. Handles failures and retries.
     int priority = (vertexDistanceFromRoot + 1) * 3;
     
-    if(currentShuffler == vertex) {
-      if(currentPartitioner != null) {
-        // special priority for current reducers while current partitioners are 
-        // still running. Schedule at priority one higher than natural priority 
-        // of previous vertex.
-        priority -= 4;  // this == (partitionerDepth+1)*3 - 1     
-      }
+    if(reOrderPriority) {
+      // special priority for current reducers while current partitioners are 
+      // still running. Schedule at priority one higher than natural priority 
+      // of previous vertex.
+      priority -= 4;  // this == (partitionerDepth+1)*3 - 1     
     } else {
       if(attempt.getIsRescheduled()) {
         // higher priority for retries of failed attempts. Only makes sense in

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
index 607e5af..9e8729e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
@@ -50,7 +50,7 @@ public class DAGSchedulerNaturalOrder implements DAGScheduler {
   @Override
   public void scheduleTask(DAGEventSchedulerUpdate event) {
     TaskAttempt attempt = event.getAttempt();
-    Vertex vertex = dag.getVertex(attempt.getID().getTaskID().getVertexID());
+    Vertex vertex = dag.getVertex(attempt.getVertexID());
     int vertexDistanceFromRoot = vertex.getDistanceFromRoot();
 
     // natural priority. Handles failures and retries.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 5d4242d..5225ecf 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -90,8 +90,10 @@ import org.apache.tez.dag.app.rm.AMSchedulerEventTALaunchRequest;
 import org.apache.tez.dag.history.DAGHistoryEvent;
 import org.apache.tez.dag.history.events.TaskAttemptFinishedEvent;
 import org.apache.tez.dag.history.events.TaskAttemptStartedEvent;
+import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.dag.utils.TezBuilderUtils;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
 
@@ -290,6 +292,16 @@ public class TaskAttemptImpl implements TaskAttempt,
     return attemptId;
   }
   
+  @Override
+  public TezVertexID getVertexID() {
+    return attemptId.getTaskID().getVertexID();
+  }
+  
+  @Override
+  public TezDAGID getDAGID() {
+    return getVertexID().getDAGId();
+  }
+  
   TezTaskContext createRemoteTask() {
     Vertex vertex = getTask().getVertex();
     DAG dag = vertex.getDAG();
@@ -601,7 +613,7 @@ public class TaskAttemptImpl implements TaskAttempt,
       TaskAttemptImpl ta) {
     DAGEventCounterUpdate jce = 
         new DAGEventCounterUpdate(
-            ta.getID().getTaskID().getVertexID().getDAGId()
+            ta.getDAGID()
             );
     jce.addCounterUpdate(DAGCounter.TOTAL_LAUNCHED_TASKS, 1);
     return jce;
@@ -611,7 +623,7 @@ public class TaskAttemptImpl implements TaskAttempt,
       TaskAttemptImpl ta) {
     DAGEventCounterUpdate jce = 
         new DAGEventCounterUpdate(            
-            ta.getID().getTaskID().getVertexID().getDAGId()
+            ta.getDAGID()
             );
 
     long slotMillis = computeSlotMillis(ta);
@@ -624,7 +636,7 @@ public class TaskAttemptImpl implements TaskAttempt,
       TaskAttemptStateInternal taState) {
     DAGEventCounterUpdate jce = 
         new DAGEventCounterUpdate(
-            taskAttempt.getID().getTaskID().getVertexID().getDAGId());
+            taskAttempt.getDAGID());
 
     long slotMillisIncrement = computeSlotMillis(taskAttempt);
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index c61cbcc..8b98c68 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -461,7 +461,7 @@ public class TaskSchedulerEventHandler extends AbstractService
 
       sendEvent(new AMContainerEventLaunchRequest(
           containerId,
-          taskAttempt.getID().getTaskID().getVertexID(),
+          taskAttempt.getVertexID(),
           event.getJobToken(),
           // TODO getConf from AMSchedulerEventTALaunchRequest
           event.getCredentials(), false, event.getConf(),

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
new file mode 100644
index 0000000..988a116
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.DAGScheduler;
+import org.apache.tez.dag.app.dag.TaskAttempt;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdate;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.mockito.Mockito.*;
+
+public class TestDAGScheduler {
+
+  class MockEventHandler implements EventHandler<TaskAttemptEventSchedule> {
+    TaskAttemptEventSchedule event;
+    @Override
+    public void handle(TaskAttemptEventSchedule event) {
+      this.event = event;
+    }
+    
+  }
+  
+  MockEventHandler mockEventHandler = new MockEventHandler();
+  
+  //@Test(timeout=10000)
+  public void testDAGSchedulerNaturalOrder() {
+    DAG mockDag = mock(DAG.class);
+    Vertex mockVertex = mock(Vertex.class);
+    TaskAttempt mockAttempt = mock(TaskAttempt.class);
+    when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex);
+    when(mockVertex.getDistanceFromRoot()).thenReturn(0).thenReturn(1)
+        .thenReturn(2);
+    when(mockAttempt.getIsRescheduled()).thenReturn(false);
+    
+    DAGEventSchedulerUpdate event = new DAGEventSchedulerUpdate(
+        DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt);    
+    
+    DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag,
+        mockEventHandler);
+    scheduler.scheduleTask(event);
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 2);
+    scheduler.scheduleTask(event);
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 4);
+    scheduler.scheduleTask(event);
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+    
+    when(mockAttempt.getIsRescheduled()).thenReturn(true);
+    scheduler.scheduleTask(event);
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 5);
+  }
+  
+  @Test//(timeout=10000)
+  public void testDAGSchedulerMRR() {
+    DAG mockDag = mock(DAG.class);
+    TezDAGID dagId = new TezDAGID("1", 1, 1);
+
+    Vertex mockVertex1 = mock(Vertex.class);
+    TezVertexID mockVertexId1 = new TezVertexID(dagId, 1);
+    when(mockVertex1.getVertexId()).thenReturn(mockVertexId1);
+    when(mockVertex1.getDistanceFromRoot()).thenReturn(0);
+    TaskAttempt mockAttempt1 = mock(TaskAttempt.class);
+    when(mockAttempt1.getVertexID()).thenReturn(mockVertexId1);
+    when(mockAttempt1.getIsRescheduled()).thenReturn(false);
+    when(mockDag.getVertex(mockVertexId1)).thenReturn(mockVertex1);
+    
+    Vertex mockVertex2 = mock(Vertex.class);
+    TezVertexID mockVertexId2 = new TezVertexID(dagId, 2);
+    when(mockVertex2.getVertexId()).thenReturn(mockVertexId2);
+    when(mockVertex2.getDistanceFromRoot()).thenReturn(1);
+    TaskAttempt mockAttempt2 = mock(TaskAttempt.class);
+    when(mockAttempt2.getVertexID()).thenReturn(mockVertexId2);
+    when(mockAttempt2.getIsRescheduled()).thenReturn(false);
+    when(mockDag.getVertex(mockVertexId2)).thenReturn(mockVertex2);
+    TaskAttempt mockAttempt2f = mock(TaskAttempt.class);
+    when(mockAttempt2f.getVertexID()).thenReturn(mockVertexId2);
+    when(mockAttempt2f.getIsRescheduled()).thenReturn(true);
+    
+    Vertex mockVertex3 = mock(Vertex.class);
+    TezVertexID mockVertexId3 = new TezVertexID(dagId, 3);
+    when(mockVertex3.getVertexId()).thenReturn(mockVertexId3);
+    when(mockVertex3.getDistanceFromRoot()).thenReturn(2);
+    TaskAttempt mockAttempt3 = mock(TaskAttempt.class);
+    when(mockAttempt3.getVertexID()).thenReturn(mockVertexId3);
+    when(mockAttempt3.getIsRescheduled()).thenReturn(false);
+    when(mockDag.getVertex(mockVertexId3)).thenReturn(mockVertex3);
+
+    DAGEventSchedulerUpdate mockEvent1 = mock(DAGEventSchedulerUpdate.class);
+    when(mockEvent1.getAttempt()).thenReturn(mockAttempt1);
+    DAGEventSchedulerUpdate mockEvent2 = mock(DAGEventSchedulerUpdate.class);
+    when(mockEvent2.getAttempt()).thenReturn(mockAttempt2);
+    DAGEventSchedulerUpdate mockEvent2f = mock(DAGEventSchedulerUpdate.class);
+    when(mockEvent2f.getAttempt()).thenReturn(mockAttempt2f);
+    DAGEventSchedulerUpdate mockEvent3 = mock(DAGEventSchedulerUpdate.class);
+    when(mockEvent3.getAttempt()).thenReturn(mockAttempt3);
+    DAGScheduler scheduler = new DAGSchedulerMRR(mockDag, mockEventHandler);
+    
+    // M starts. M completes. R1 starts. R1 completes. R2 starts. R2 completes
+    scheduler.scheduleTask(mockEvent1); // M starts
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+    scheduler.scheduleTask(mockEvent1); // M runs another
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+    scheduler.vertexCompleted(mockVertex1); // M completes
+    scheduler.scheduleTask(mockEvent2); // R1 starts
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+    scheduler.scheduleTask(mockEvent2); // R1 runs another
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+    scheduler.scheduleTask(mockEvent2f); // R1 runs retry. Retry priority
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 4);
+    scheduler.vertexCompleted(mockVertex2); // R1 completes
+    scheduler.scheduleTask(mockEvent3); // R2 starts
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 9);
+    scheduler.scheduleTask(mockEvent3); // R2 runs another
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 9);
+    scheduler.vertexCompleted(mockVertex3); // R2 completes  
+    
+    // M starts. R1 starts. M completes. R2 starts. R1 completes. R2 completes
+    scheduler.scheduleTask(mockEvent1); // M starts
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+    scheduler.scheduleTask(mockEvent2); // R1 starts. Reordered priority
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 2);
+    scheduler.scheduleTask(mockEvent1); // M runs another
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+    scheduler.scheduleTask(mockEvent2); // R1 runs another. Reordered priority
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 2);
+    scheduler.scheduleTask(mockEvent2f); // R1 runs retry. Reordered priority
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 2);
+    scheduler.vertexCompleted(mockVertex1); // M completes
+    scheduler.scheduleTask(mockEvent3); // R2 starts. Reordered priority
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 5);
+    scheduler.scheduleTask(mockEvent2); // R1 runs another. Normal priority
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+    scheduler.scheduleTask(mockEvent2f); // R1 runs retry. Retry priority
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 4);
+    scheduler.scheduleTask(mockEvent3); // R2 runs another. Reordered priority
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 5);
+    scheduler.vertexCompleted(mockVertex2); // R1 completes
+    scheduler.vertexCompleted(mockVertex3); // R2 completes      
+    
+    // M starts. M completes. R1 starts. R2 starts. R1 completes. R2 completes
+    scheduler.scheduleTask(mockEvent1); // M starts
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+    scheduler.vertexCompleted(mockVertex1); // M completes
+    scheduler.scheduleTask(mockEvent2); // R1 starts
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+    scheduler.scheduleTask(mockEvent3); // R2 starts. Reordered priority
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 5);
+    scheduler.scheduleTask(mockEvent2); // R1 runs another
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+    scheduler.vertexCompleted(mockVertex2); // R1 completes
+    scheduler.vertexCompleted(mockVertex3); // R2 completes
+    
+    // M starts. R1 starts. M completes. R1 completes. R2 starts. R2 completes
+    scheduler.scheduleTask(mockEvent1); // M starts
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+    scheduler.scheduleTask(mockEvent2); // R1 starts. Reordered priority
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 2);
+    scheduler.vertexCompleted(mockVertex1); // M completes
+    scheduler.scheduleTask(mockEvent2); // R1 starts. Normal priority
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+    scheduler.vertexCompleted(mockVertex2); // R1 completes
+    scheduler.scheduleTask(mockEvent3); // R2 starts
+    Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 9);
+    scheduler.vertexCompleted(mockVertex3); // R2 completes  
+  }
+}


[07/50] [abbrv] TEZ-123. Cleanup package dependencies. (bikas)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
index f9e8eb7..6c7bb0f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
@@ -29,9 +29,9 @@ import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
 import org.apache.tez.dag.app.dag.Vertex;
 import org.apache.tez.dag.app.dag.VertexScheduler;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 
 /**
  * Starts scheduling tasks when number of completed source tasks crosses 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index cc3170e..02e91b4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -58,8 +58,8 @@ import org.apache.tez.dag.api.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint;
-import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
+import org.apache.tez.dag.api.impl.DAGStatus;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
@@ -81,12 +81,12 @@ import org.apache.tez.dag.app.dag.event.VertexEventType;
 import org.apache.tez.dag.history.DAGHistoryEvent;
 import org.apache.tez.dag.history.events.DAGFinishedEvent;
 import org.apache.tez.dag.history.events.DAGStartedEvent;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.dag.utils.DAGApps;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
 import org.apache.tez.engine.common.security.JobTokenSecretManager;
 import org.apache.tez.engine.common.security.TokenCache;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezVertexID;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 /** Implementation of Job interface. Maintains the state machines of Job.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexScheduler.java
index 1d13aa4..bd39115 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexScheduler.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.app.dag.impl;
 
 import org.apache.tez.dag.app.dag.Vertex;
 import org.apache.tez.dag.app.dag.VertexScheduler;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 /**
  * Starts all tasks immediately on vertex start

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index c907e48..2564ffa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -91,9 +91,9 @@ import org.apache.tez.dag.app.rm.AMSchedulerEventTALaunchRequest;
 import org.apache.tez.dag.history.DAGHistoryEvent;
 import org.apache.tez.dag.history.events.TaskAttemptFinishedEvent;
 import org.apache.tez.dag.history.events.TaskAttemptStartedEvent;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
 
 import com.google.common.annotations.VisibleForTesting;
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index cf49dbd..4d0c4f6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -67,11 +67,11 @@ import org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule;
 import org.apache.tez.dag.history.DAGHistoryEvent;
 import org.apache.tez.dag.history.events.TaskFinishedEvent;
 import org.apache.tez.dag.history.events.TaskStartedEvent;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
 import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
 
 import com.google.common.annotations.VisibleForTesting;
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index b9e911c..3f3ac49 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -62,11 +62,11 @@ import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.client.VertexStatus;
-import org.apache.tez.dag.api.client.VertexStatus.State;
 import org.apache.tez.dag.api.impl.NullVertexOutputCommitter;
 import org.apache.tez.dag.api.impl.VertexContext;
 import org.apache.tez.dag.api.impl.VertexOutputCommitter;
+import org.apache.tez.dag.api.impl.VertexStatus;
+import org.apache.tez.dag.api.impl.VertexStatus.State;
 import org.apache.tez.dag.api.records.TaskState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
@@ -94,12 +94,12 @@ import org.apache.tez.dag.app.dag.event.VertexEventType;
 import org.apache.tez.dag.history.DAGHistoryEvent;
 import org.apache.tez.dag.history.events.VertexFinishedEvent;
 import org.apache.tez.dag.history.events.VertexStartedEvent;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezDAGID;
 import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
 
 
 /** Implementation of Vertex interface. Maintains the state machines of Vertex.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
index 3a32662..fddd204 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.app.rm;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.tez.dag.api.records.TaskAttemptState;
 import org.apache.tez.dag.app.dag.TaskAttempt;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class AMSchedulerEventTAEnded extends AMSchedulerEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
index 04fd740..b595c2b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
@@ -27,8 +27,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.tez.common.TezTaskContext;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.app.dag.TaskAttempt;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 
 public class AMSchedulerEventTALaunchRequest extends AMSchedulerEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
index 3637ded..3c00620 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
@@ -23,7 +23,7 @@ import java.util.List;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public interface AMContainer extends EventHandler<AMContainerEvent>{
   

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
index 7bc0d7e..0fe5b3b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
@@ -19,7 +19,7 @@ package org.apache.tez.dag.app.rm.container;
 
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.tez.common.TezTaskContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class AMContainerEventAssignTA extends AMContainerEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
index e672052..4e199da 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
@@ -25,9 +25,9 @@ import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezVertexID;
 
 public class AMContainerEventLaunchRequest extends AMContainerEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java
index 1deb9f4..e0c8fe7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java
@@ -18,7 +18,7 @@
 package org.apache.tez.dag.app.rm.container;
 
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class AMContainerEventTASucceeded extends AMContainerEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index c02ed7a..5eca572 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -49,11 +49,11 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.dag.utils.TezEngineChildJVM;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
 import org.apache.tez.engine.common.security.TokenCache;
 import org.apache.tez.engine.common.shuffle.server.ShuffleHandler;
-import org.apache.tez.engine.records.TezVertexID;
 
 import com.google.common.annotations.VisibleForTesting;
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 474af31..1089013 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -56,7 +56,7 @@ import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
 import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
 import org.apache.tez.dag.history.DAGHistoryEvent;
 import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 @SuppressWarnings("rawtypes")
 public class AMContainerImpl implements AMContainer {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
index 9173713..cc2b7f7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
@@ -19,7 +19,7 @@ package org.apache.tez.dag.app.rm.node;
 
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class AMNodeEventTaskAttemptEnded extends AMNodeEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java
index 6599efa..b07d594 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java
@@ -19,7 +19,7 @@ package org.apache.tez.dag.app.rm.node;
 
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class AMNodeEventTaskAttemptSucceeded extends AMNodeEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
index ccd148c..04c104a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
@@ -47,9 +47,9 @@ import org.apache.tez.dag.app.dag.TaskAttempt;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
 import org.apache.tez.dag.app.dag.event.TaskEvent;
 import org.apache.tez.dag.app.dag.event.TaskEventType;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 // FIXME does not handle multiple vertices

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java
index 83196b0..558f78c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
index 4ef9991..aa95f37 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
@@ -28,8 +28,8 @@ import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.Task;
 import org.apache.tez.dag.app.dag.TaskAttempt;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
 
 public class LegacyTaskRuntimeEstimator extends StartEndTimesBase {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/NullTaskRuntimesEngine.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/NullTaskRuntimesEngine.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/NullTaskRuntimesEngine.java
index e49c9fb..9907bc6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/NullTaskRuntimesEngine.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/NullTaskRuntimesEngine.java
@@ -21,8 +21,8 @@ package org.apache.tez.dag.app.speculate;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
 
 
 /*

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/SpeculatorEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/SpeculatorEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/SpeculatorEvent.java
index 08697ee..c8f400b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/SpeculatorEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/SpeculatorEvent.java
@@ -20,9 +20,9 @@ package org.apache.tez.dag.app.speculate;
 
 import org.apache.hadoop.yarn.event.AbstractEvent;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
 
 public class SpeculatorEvent extends AbstractEvent<Speculator.EventType> {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
index 317d9c1..191d0b9 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
@@ -33,9 +33,9 @@ import org.apache.tez.dag.app.dag.Task;
 import org.apache.tez.dag.app.dag.TaskAttempt;
 import org.apache.tez.dag.app.dag.Vertex;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 abstract class StartEndTimesBase<V> implements TaskRuntimeEstimator {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskRuntimeEstimator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskRuntimeEstimator.java
index 81aa9c9..965d965 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskRuntimeEstimator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskRuntimeEstimator.java
@@ -21,8 +21,8 @@ package org.apache.tez.dag.app.speculate;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskSpeculationPredicate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskSpeculationPredicate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskSpeculationPredicate.java
index 6516bac..33ea09f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskSpeculationPredicate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskSpeculationPredicate.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.app.speculate;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.Task;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
 
 
 public class TaskSpeculationPredicate {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanupEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanupEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanupEvent.java
index 1839f53..9b6cc15 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanupEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanupEvent.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.app.taskclean;
 import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 /**
  * This class encapsulates task cleanup event.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java
index ba9d258..d3b8a91 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.history;
 
 import org.apache.hadoop.yarn.event.AbstractEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
 
 public class DAGHistoryEvent extends AbstractEvent<HistoryEventType>{
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
index 547a09a..d1b87b1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
@@ -18,11 +18,11 @@
 
 package org.apache.tez.dag.history.events;
 
-import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.impl.DAGStatus;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.DAGFinished;
 import org.apache.tez.dag.history.avro.HistoryEventType;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
 
 public class DAGFinishedEvent implements HistoryEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java
index 104e949..719538c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.history.events;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.DAGStarted;
 import org.apache.tez.dag.history.avro.HistoryEventType;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
 
 public class DAGStartedEvent implements HistoryEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
index 7b5f3a1..4a2917e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
@@ -22,7 +22,7 @@ import org.apache.tez.dag.api.records.TaskAttemptState;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
 import org.apache.tez.dag.history.avro.TaskAttemptFinished;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskAttemptFinishedEvent implements HistoryEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
index ea3e71f..97c9f2f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
 import org.apache.tez.dag.history.avro.TaskAttemptStarted;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskAttemptStartedEvent implements HistoryEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
index 270e50a..165dc20 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
@@ -22,7 +22,7 @@ import org.apache.tez.dag.api.records.TaskState;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
 import org.apache.tez.dag.history.avro.TaskFinished;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
 
 public class TaskFinishedEvent implements HistoryEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java
index 6ed7868..964cc79 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.history.events;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
 import org.apache.tez.dag.history.avro.TaskStarted;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
 
 public class TaskStartedEvent implements HistoryEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
index b6d4d0b..b02be1e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
@@ -18,11 +18,11 @@
 
 package org.apache.tez.dag.history.events;
 
-import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.impl.VertexStatus;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
 import org.apache.tez.dag.history.avro.VertexFinished;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezVertexID;
 
 public class VertexFinishedEvent implements HistoryEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java
index 98367dd..a4fa20e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.history.events;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
 import org.apache.tez.dag.history.avro.VertexStarted;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezVertexID;
 
 public class VertexStartedEvent implements HistoryEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
index fee0846..727ded5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
@@ -30,7 +30,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.app.rm.container.AMContainerHelpers;
+import org.apache.tez.dag.records.TezVertexID;
 
 public class TezEngineChildJVM {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index 12ddcf0..a18bd3b 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -66,11 +66,11 @@ import org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
 import org.apache.tez.dag.app.rm.container.AMContainerMap;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
 import org.junit.Test;
 
 @SuppressWarnings({ "unchecked", "rawtypes" })

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 13a19b9..516ea5e 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -52,11 +52,11 @@ import org.apache.tez.dag.app.dag.TaskStateInternal;
 import org.apache.tez.dag.app.dag.event.TaskEvent;
 import org.apache.tez.dag.app.dag.event.TaskEventTAUpdate;
 import org.apache.tez.dag.app.dag.event.TaskEventType;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
 import org.junit.Before;
 import org.junit.Test;
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-engine-api/pom.xml b/tez-engine-api/pom.xml
index 7a747d2..2cc7276 100644
--- a/tez-engine-api/pom.xml
+++ b/tez-engine-api/pom.xml
@@ -25,6 +25,10 @@
   <artifactId>tez-engine-api</artifactId>
   <dependencies>
     <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-common</artifactId>
+    </dependency>
+    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/api/Master.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/api/Master.java b/tez-engine-api/src/main/java/org/apache/tez/engine/api/Master.java
index 3358a03..f3add9a 100644
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/api/Master.java
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/api/Master.java
@@ -19,7 +19,7 @@ package org.apache.tez.engine.api;
 
 import org.apache.hadoop.ipc.ProtocolInfo;
 import org.apache.hadoop.ipc.VersionedProtocol;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDAGID.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDAGID.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDAGID.java
deleted file mode 100644
index 1066409..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDAGID.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.records;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.text.NumberFormat;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.BuilderUtils;
-
-
-/**
- * TezDAGID represents the immutable and unique identifier for
- * a Tez DAG.
- *
- * TezDAGID consists of 2 parts. The first part is the {@link ApplicationId},
- * that is the YARN Application ID that this DAG belongs to. The second part is
- * the DAG number.
- *
- * @see ApplicationId
- */
-public class TezDAGID extends TezID {
-
-  public static final String DAG = "dag";
-  protected static final NumberFormat idFormat = NumberFormat.getInstance();
-  static {
-    idFormat.setGroupingUsed(false);
-    idFormat.setMinimumIntegerDigits(6);
-  }
-
-  private ApplicationId applicationId;
-
-  public TezDAGID() {
-  }
-
-  /**
-   * Constructs a DAGID object from given {@link ApplicationId}.
-   * @param applicationId Application that this dag belongs to
-   * @param id the dag number
-   */
-  public TezDAGID(ApplicationId applicationId, int id) {
-    super(id);
-    if(applicationId == null) {
-      throw new IllegalArgumentException("applicationId cannot be null");
-    }
-    this.applicationId = applicationId;
-  }
-
-  /**
-   * Constructs a DAGID object from given parts.
-   * @param yarnRMIdentifier YARN RM identifier
-   * @param applicationId application number
-   * @param id the dag number
-   */
-  public TezDAGID(String yarnRMIdentifier, int appId, int id) {
-    this(BuilderUtils.newApplicationId(Long.valueOf(yarnRMIdentifier),
-        appId), id);
-  }
-
-  /** Returns the {@link ApplicationId} object that this dag belongs to */
-  public ApplicationId getApplicationId() {
-    return applicationId;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (!super.equals(o))
-      return false;
-
-    TezDAGID that = (TezDAGID)o;
-    return this.applicationId.equals(that.applicationId);
-  }
-
-  /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
-  @Override
-  public int compareTo(TezID o) {
-    TezDAGID that = (TezDAGID)o;
-    return this.applicationId.compareTo(that.applicationId);
-  }
-
-  @Override
-  public String toString() {
-    return appendTo(new StringBuilder(DAG)).toString();
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    applicationId = BuilderUtils.newApplicationId(in.readLong(), in.readInt());
-    super.readFields(in);
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeLong(applicationId.getClusterTimestamp());
-    out.writeInt(applicationId.getId());
-    super.write(out);
-  }
-
-  /**
-   * Add the unique string to the given builder.
-   * @param builder the builder to append to
-   * @return the builder that was passed in
-   */
-  protected StringBuilder appendTo(StringBuilder builder) {
-    return builder.append(SEPARATOR).
-                 append(applicationId.getClusterTimestamp()).
-                 append(SEPARATOR).
-                 append(applicationId.getId()).
-                 append(SEPARATOR).
-                 append(idFormat.format(id));
-  }
-
-  @Override
-  public int hashCode() {
-    return applicationId.hashCode() * 524287 + id;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDependentTaskCompletionEvent.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDependentTaskCompletionEvent.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDependentTaskCompletionEvent.java
index 91ce53c..cba752e 100644
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDependentTaskCompletionEvent.java
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDependentTaskCompletionEvent.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 /**
  * This is used to track task completion events on 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezID.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezID.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezID.java
deleted file mode 100644
index 13ea05a..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezID.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.records;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.WritableComparable;
-
-/**
- * A general identifier, which internally stores the id
- * as an integer. This is the super class of {@link TezJobID}, 
- * {@link TezTaskID} and {@link TezTaskAttemptID}.
- * 
- * @see TezTaskID
- * @see TezTaskAttemptID
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public abstract class TezID implements WritableComparable<TezID> {
-  public static final char SEPARATOR = '_';
-  protected int id;
-
-  /** constructs an ID object from the given int */
-  public TezID(int id) {
-    this.id = id;
-  }
-
-  protected TezID() {
-  }
-
-  /** returns the int which represents the identifier */
-  public int getId() {
-    return id;
-  }
-
-  @Override
-  public String toString() {
-    return String.valueOf(id);
-  }
-
-  @Override
-  public int hashCode() {
-    return id;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o)
-      return true;
-    if(o == null)
-      return false;
-    if (o.getClass() == this.getClass()) {
-      TezID that = (TezID) o;
-      return this.id == that.id;
-    }
-    else
-      return false;
-  }
-
-  /** Compare IDs by associated numbers */
-  public int compareTo(TezID that) {
-    return this.id - that.id;
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    this.id = in.readInt();
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(id);
-  }
-  
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskAttemptID.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskAttemptID.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskAttemptID.java
deleted file mode 100644
index e59c5bf..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskAttemptID.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.records;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * TezTaskAttemptID represents the immutable and unique identifier for
- * a task attempt. Each task attempt is one particular instance of a Tez Task
- * identified by its TezTaskID.
- *
- * TezTaskAttemptID consists of 2 parts. First part is the
- * {@link TezTaskID}, that this TaskAttemptID belongs to.
- * Second part is the task attempt number. <br>
- * <p>
- * Applications should never construct or parse TaskAttemptID strings
- * , but rather use appropriate constructors or {@link #forName(String)}
- * method.
- *
- * @see TezTaskID
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class TezTaskAttemptID extends TezID {
-  public static final String ATTEMPT = "attempt";
-  private TezTaskID taskId;
-  
-  public TezTaskAttemptID() {
-    taskId = new TezTaskID();
-  }
-  
-  /**
-   * Constructs a TaskAttemptID object from given {@link TezTaskID}.  
-   * @param taskId TaskID that this task belongs to  
-   * @param id the task attempt number
-   */
-  public TezTaskAttemptID(TezTaskID taskId, int id) {
-    super(id);
-    if(taskId == null) {
-      throw new IllegalArgumentException("taskId cannot be null");
-    }
-    this.taskId = taskId;
-  }
-
-  /** Returns the {@link TezTaskID} object that this task attempt belongs to */
-  public TezTaskID getTaskID() {
-    return taskId;
-  }
-  
-  @Override
-  public boolean equals(Object o) {
-    if (!super.equals(o))
-      return false;
-
-    TezTaskAttemptID that = (TezTaskAttemptID)o;
-    return this.taskId.equals(that.taskId);
-  }
-  
-  /**
-   * Add the unique string to the StringBuilder
-   * @param builder the builder to append ot
-   * @return the builder that was passed in.
-   */
-  protected StringBuilder appendTo(StringBuilder builder) {
-    return taskId.appendTo(builder).append(SEPARATOR).append(id);
-  }
-  
-  @Override
-  public int hashCode() {
-    return taskId.hashCode() * 539501 + id;
-  }
-  
-  /**Compare TaskIds by first tipIds, then by task numbers. */
-  @Override
-  public int compareTo(TezID o) {
-    TezTaskAttemptID that = (TezTaskAttemptID)o;
-    int tipComp = this.taskId.compareTo(that.taskId);
-    if(tipComp == 0) {
-      return this.id - that.id;
-    }
-    else return tipComp;
-  }
-  @Override
-  public String toString() { 
-    return appendTo(new StringBuilder(ATTEMPT)).toString();
-  }
-  
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    taskId.readFields(in);
-    super.readFields(in);
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    taskId.write(out);
-    super.write(out);
-  }
-
-  // FIXME TEZ DAG needs to be removed
-  public static TezTaskAttemptID read(DataInput in) throws IOException {
-    TezTaskAttemptID tId = new TezTaskAttemptID();
-    tId.readFields(in);
-    return tId;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskID.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskID.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskID.java
deleted file mode 100644
index b6a997d..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskID.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.records;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.text.NumberFormat;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-
-/**
- * TaskID represents the immutable and unique identifier for 
- * a Tez Task. Each TaskID encompasses multiple attempts made to
- * execute the Tez Task, each of which are uniquely identified by
- * their TezTaskAttemptID.
- * 
- * @see TezTaskAttemptID
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class TezTaskID extends TezID {
-  public static final String TASK = "task";
-  protected static final NumberFormat idFormat = NumberFormat.getInstance();
-  static {
-    idFormat.setGroupingUsed(false);
-    idFormat.setMinimumIntegerDigits(6);
-  }
-  
-  private TezVertexID vertexId;
-  
-  public TezTaskID() {
-    vertexId = new TezVertexID();
-  }
-  
-  /**
-   * Constructs a TaskID object from given {@link MRxApplicationID}.  
-   * @param jobId JobID that this tip belongs to 
-   * @param type the {@link TezTaskType} of the task 
-   * @param id the tip number
-   */
-  public TezTaskID(TezVertexID vertexId, int id) {
-    super(id);
-    if(vertexId == null) {
-      throw new IllegalArgumentException("vertexId cannot be null");
-    }
-    this.vertexId = vertexId;
-  }
-  
-  /** Returns the {@link TezVertexID} object that this task belongs to */
-  public TezVertexID getVertexID() {
-    return vertexId;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (!super.equals(o))
-      return false;
-
-    TezTaskID that = (TezTaskID)o;
-    return this.vertexId.equals(that.vertexId);
-  }
-
-  /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
-  @Override
-  public int compareTo(TezID o) {
-    TezTaskID that = (TezTaskID)o;
-    int vertexComp = this.vertexId.compareTo(that.vertexId);
-    if(vertexComp == 0) {
-      return this.id - that.id;
-    }
-    else return vertexComp;
-  }
-  @Override
-  public String toString() { 
-    return appendTo(new StringBuilder(TASK)).toString();
-  }
-
-  /**
-   * Add the unique string to the given builder.
-   * @param builder the builder to append to
-   * @return the builder that was passed in
-   */
-  protected StringBuilder appendTo(StringBuilder builder) {
-    return vertexId.appendTo(builder).
-                 append(SEPARATOR).
-                 append(idFormat.format(id));
-  }
-  
-  @Override
-  public int hashCode() {
-    return vertexId.hashCode() * 535013 + id;
-  }
-  
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    vertexId.readFields(in);
-    super.readFields(in);
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    vertexId.write(out);
-    super.write(out);
-  }
-  
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskType.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskType.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskType.java
deleted file mode 100644
index 68a5541..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskType.java
+++ /dev/null
@@ -1,26 +0,0 @@
-package org.apache.tez.engine.records;
-///**
-// * Licensed to the Apache Software Foundation (ASF) under one or more
-// * contributor license agreements. See the NOTICE file distributed with this
-// * work for additional information regarding copyright ownership. The ASF
-// * licenses this file to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// * 
-// * http://www.apache.org/licenses/LICENSE-2.0
-// * 
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// * License for the specific language governing permissions and limitations under
-// * the License.
-// */
-//
-//package org.apache.tez.records;
-//
-//
-//public interface TezTaskType {
-//  
-//  public String toSerializedString();
-//
-//}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezVertexID.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezVertexID.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezVertexID.java
deleted file mode 100644
index 31acd56..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezVertexID.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.records;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.text.NumberFormat;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * TezVertexID represents the immutable and unique identifier for
- * a Vertex in a Tez DAG. Each TezVertexID encompasses multiple Tez Tasks.
- *
- * TezVertezID consists of 2 parts. The first part is the {@link TezDAGID},
- * that is the Tez DAG that this vertex belongs to. The second part is
- * the vertex number.
- *
- * @see TezDAGID
- * @see TezTaskID
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class TezVertexID extends TezID {
-  public static final String VERTEX = "vertex";
-  protected static final NumberFormat idFormat = NumberFormat.getInstance();
-  static {
-    idFormat.setGroupingUsed(false);
-    idFormat.setMinimumIntegerDigits(6);
-  }
-  
-  private TezDAGID dagId;
-  
-  public TezVertexID() {
-  }
-  
-  /**
-   * Constructs a TaskID object from given {@link TezDAGID}.
-   * @param applicationId JobID that this tip belongs to
-   * @param type the {@link TezTaskType} of the task
-   * @param id the tip number
-   */
-  public TezVertexID(TezDAGID dagId, int id) {
-    super(id);
-    if(dagId == null) {
-      throw new IllegalArgumentException("dagId cannot be null");
-    }
-    this.dagId = dagId;
-  }
-  
-  /** Returns the {@link TezDAGID} object that this tip belongs to */
-  public TezDAGID getDAGId() {
-    return dagId;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (!super.equals(o))
-      return false;
-
-    TezVertexID that = (TezVertexID)o;
-    return this.dagId.equals(that.dagId);
-  }
-
-  /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
-  @Override
-  public int compareTo(TezID o) {
-    TezVertexID that = (TezVertexID)o;
-    return this.dagId.compareTo(that.dagId);
-  }
-  
-  @Override
-  public String toString() { 
-    return appendTo(new StringBuilder(VERTEX)).toString();
-  }
-  
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    dagId = new TezDAGID();
-    dagId.readFields(in);
-    super.readFields(in);
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    dagId.write(out);
-    super.write(out);
-  }
-
-  /**
-   * Add the unique string to the given builder.
-   * @param builder the builder to append to
-   * @return the builder that was passed in
-   */
-  protected StringBuilder appendTo(StringBuilder builder) {
-    return builder.append(SEPARATOR).
-        append(dagId.getApplicationId().getClusterTimestamp()).
-        append(SEPARATOR).
-        append(dagId.getApplicationId().getId()).
-        append(SEPARATOR).
-        append(dagId.getId()).
-        append(SEPARATOR).
-        append(idFormat.format(id));
-  }
-  
-  @Override
-  public int hashCode() {
-    return dagId.hashCode() * 530017 + id;
-  }
-  
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/pom.xml
----------------------------------------------------------------------
diff --git a/tez-engine/pom.xml b/tez-engine/pom.xml
index ba5b6e6..07bda8c 100644
--- a/tez-engine/pom.xml
+++ b/tez-engine/pom.xml
@@ -34,6 +34,10 @@
       <artifactId>tez-common</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-engine-api</artifactId>
+    </dependency>
+    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-common</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
index 7f825d9..350702a 100644
--- a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
+++ b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
@@ -25,7 +25,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.hadoop.io.Text;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TezEngineTaskContext extends TezTaskContext {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/common/TezTaskReporter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/common/TezTaskReporter.java b/tez-engine/src/main/java/org/apache/tez/common/TezTaskReporter.java
new file mode 100644
index 0000000..1931e31
--- /dev/null
+++ b/tez-engine/src/main/java/org/apache/tez/common/TezTaskReporter.java
@@ -0,0 +1,116 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.common;
+
+import java.io.IOException;
+
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.util.Progressable;
+import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.engine.api.Master;
+import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
+
+public interface TezTaskReporter extends Progressable, Master {
+
+  public void setStatus(String status);
+
+  public float getProgress();
+
+  public void setProgress(float progress);
+  
+  public void progress();
+
+  public TezCounter getCounter(String group, String name);
+
+  public TezCounter getCounter(Enum<?> name);
+
+  public void incrCounter(String group, String counter, long amount);
+
+  public void incrCounter(Enum<?> key, long amount);
+
+  public void reportFatalError(TezTaskAttemptID taskAttemptId, 
+      Throwable exception, String logMsg);
+
+  public final TezTaskReporter NULL = new TezTaskReporter() {
+
+    @Override
+    public TezTaskDependencyCompletionEventsUpdate getDependentTasksCompletionEvents(
+        int fromEventIdx, int maxEventsToFetch,
+        TezTaskAttemptID reduce) {
+      return null;
+    }
+    
+    @Override
+    public void setStatus(String status) {
+    }
+    
+    @Override
+    public void setProgress(float progress) {
+    }
+    
+    @Override
+    public void progress() {
+    }
+    
+    @Override
+    public void incrCounter(Enum<?> key, long amount) {
+    }
+    
+    @Override
+    public void incrCounter(String group, String counter, long amount) {
+    }
+    
+    @Override
+    public float getProgress() {
+      return 0.0f;
+    }
+    
+    @Override
+    public TezCounter getCounter(Enum<?> name) {
+      return null;
+    }
+    
+    @Override
+    public TezCounter getCounter(String group, String name) {
+      return null;
+    }
+
+    @Override
+    public void reportFatalError(TezTaskAttemptID taskAttemptId,
+        Throwable exception, String logMsg) {
+      // TODO Auto-generated method stub
+      
+    }
+
+    @Override
+    public long getProtocolVersion(String protocol, long clientVersion)
+        throws IOException {
+      // TODO TEZAM3
+      return 0;
+    }
+
+    @Override
+    public ProtocolSignature getProtocolSignature(String protocol,
+        long clientVersion, int clientMethodsHash) throws IOException {
+      // TODO TEZAM3
+      return null;
+    }
+  };
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java b/tez-engine/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
new file mode 100644
index 0000000..e94fd30
--- /dev/null
+++ b/tez-engine/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.tez.common.records.ProceedToCompletionResponse;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.engine.api.Master;
+import org.apache.tez.engine.records.OutputContext;
+
+/** Protocol that task child process uses to contact its parent process.  The
+ * parent is a daemon which which polls the central master for a new map or
+ * reduce task and runs it as a child process.  All communication between child
+ * and parent is via this protocol. */ 
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+public interface TezTaskUmbilicalProtocol extends Master {
+
+  public static final long versionID = 19L;
+
+  ContainerTask getTask(ContainerContext containerContext) throws IOException;
+  
+  boolean statusUpdate(TezTaskAttemptID taskId, TezTaskStatus taskStatus) 
+  throws IOException, InterruptedException;
+  
+  void reportDiagnosticInfo(TezTaskAttemptID taskid, String trace) throws IOException;
+  
+  boolean ping(TezTaskAttemptID taskid) throws IOException;
+
+  void done(TezTaskAttemptID taskid) throws IOException;
+  
+  void commitPending(TezTaskAttemptID taskId, TezTaskStatus taskStatus) 
+  throws IOException, InterruptedException;  
+
+  boolean canCommit(TezTaskAttemptID taskid) throws IOException;
+
+  void shuffleError(TezTaskAttemptID taskId, String message) throws IOException;
+  
+  void fsError(TezTaskAttemptID taskId, String message) throws IOException;
+
+  void fatalError(TezTaskAttemptID taskId, String message) throws IOException;
+  
+  // TODO TEZAM5 Can commitPending and outputReady be collapsed into a single
+  // call.
+  // IAC outputReady followed by commit is a little confusing - since the output
+  // isn't really in place till a commit is called. Maybe rename to
+  // processingComplete or some such.
+  
+  // TODO EVENTUALLY This is not the most useful API. Once there's some kind of
+  // support for the Task handing output over to the Container, this won't rally
+  // be required. i.e. InMemShuffle running as a service in the Container, or
+  // the second task in getTask(). ContainerUmbilical would include getTask and
+  // getServices...
+  
+  void outputReady(TezTaskAttemptID taskAttemptId, OutputContext outputContext)
+      throws IOException;
+  
+  ProceedToCompletionResponse
+      proceedToCompletion(TezTaskAttemptID taskAttemptId) throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/EventFetcher.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/EventFetcher.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/EventFetcher.java
index ac8ea27..9280423 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/EventFetcher.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/EventFetcher.java
@@ -22,9 +22,9 @@ import java.net.URI;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.api.Master;
 import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
 
 class EventFetcher extends Thread {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java
index 6f6851b..9e905e1 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java
@@ -49,11 +49,11 @@ import org.apache.tez.common.IDUtils;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezTaskReporter;
 import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.common.ConfigUtils;
 import org.apache.tez.engine.common.security.SecureShuffleUtils;
 import org.apache.tez.engine.common.shuffle.impl.MapOutput.Type;
 import org.apache.tez.engine.common.sort.impl.IFileInputStream;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 
 import com.google.common.annotations.VisibleForTesting;
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryReader.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryReader.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryReader.java
index 78db3d5..7cea558 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryReader.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryReader.java
@@ -26,9 +26,9 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.common.sort.impl.IFile;
 import org.apache.tez.engine.common.sort.impl.IFile.Reader;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 
 /**
  * <code>IFile.InMemoryReader</code> to read map-outputs present in-memory.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapHost.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapHost.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapHost.java
index 4c12e2a..544f3b5 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapHost.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapHost.java
@@ -20,7 +20,7 @@ package org.apache.tez.engine.common.shuffle.impl;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 
 class MapHost {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapOutput.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapOutput.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapOutput.java
index 41820b3..272709e 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapOutput.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapOutput.java
@@ -29,8 +29,8 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BoundedByteArrayOutputStream;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.common.task.local.output.TezTaskOutputFiles;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 
 class MapOutput {
   private static final Log LOG = LogFactory.getLog(MapOutput.class);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java
index 9156f28..16ded35 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java
@@ -45,6 +45,8 @@ import org.apache.tez.common.Constants;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezTaskReporter;
 import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
 import org.apache.tez.engine.api.Input;
 import org.apache.tez.engine.api.Output;
 import org.apache.tez.engine.api.Processor;
@@ -57,8 +59,6 @@ import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator;
 import org.apache.tez.engine.common.sort.impl.IFile.Writer;
 import org.apache.tez.engine.common.sort.impl.TezMerger.Segment;
 import org.apache.tez.engine.common.task.local.output.TezTaskOutputFiles;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
 
 @InterfaceAudience.Private
 @InterfaceStability.Unstable

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleClientMetrics.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleClientMetrics.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleClientMetrics.java
index 0188f21..34b26c4 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleClientMetrics.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleClientMetrics.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.metrics.MetricsUtil;
 import org.apache.hadoop.metrics.Updater;
 import org.apache.tez.common.Constants;
 import org.apache.tez.common.TezJobConfig;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 class ShuffleClientMetrics implements Updater {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java
index d0ce5f5..de20621 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java
@@ -39,8 +39,8 @@ import org.apache.hadoop.util.Progress;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezTaskStatus;
 import org.apache.tez.common.counters.TezCounter;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
 
 class ShuffleScheduler {
   static ThreadLocal<Long> shuffleStart = new ThreadLocal<Long>() {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
index f6af426..76fc2e9 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
@@ -45,6 +45,7 @@ import org.apache.tez.common.TezEngineTaskContext;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.counters.TaskCounter;
 import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.api.Input;
 import org.apache.tez.engine.api.Master;
 import org.apache.tez.engine.api.Output;
@@ -58,7 +59,6 @@ import org.apache.tez.engine.common.sort.impl.IFile.Writer;
 import org.apache.tez.engine.common.task.local.output.TezTaskOutput;
 import org.apache.tez.engine.common.task.local.output.TezTaskOutputFiles;
 import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 
 @SuppressWarnings({"unchecked", "rawtypes"})
 public abstract class ExternalSorter {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java
index 6ce39da..4da5ffa 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java
@@ -45,13 +45,13 @@ import org.apache.hadoop.util.IndexedSorter;
 import org.apache.hadoop.util.Progress;
 import org.apache.tez.common.TezEngineTaskContext;
 import org.apache.tez.common.TezJobConfig;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.api.Master;
 import org.apache.tez.engine.common.ConfigUtils;
 import org.apache.tez.engine.common.sort.SortingOutput;
 import org.apache.tez.engine.common.sort.impl.IFile.Writer;
 import org.apache.tez.engine.common.sort.impl.TezMerger.Segment;
 import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 
 @SuppressWarnings({"unchecked", "rawtypes"})
 public class PipelinedSorter extends ExternalSorter implements SortingOutput {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
index 0c35760..2cf5ab4 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.tez.common.TezEngineTaskContext;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezTaskContext;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.api.Master;
 import org.apache.tez.engine.common.ConfigUtils;
 import org.apache.tez.engine.common.sort.impl.ExternalSorter;
@@ -53,7 +54,6 @@ import org.apache.tez.engine.common.sort.impl.TezSpillRecord;
 import org.apache.tez.engine.common.sort.impl.IFile.Writer;
 import org.apache.tez.engine.common.sort.impl.TezMerger.Segment;
 import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 
 @SuppressWarnings({"unchecked", "rawtypes"})
 public class DefaultSorter extends ExternalSorter implements IndexedSortable {


[20/50] [abbrv] git commit: TEZ-92. Pass m/r java-opts to DAG plan

Posted by ss...@apache.org.
TEZ-92. Pass m/r java-opts to DAG plan


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/1b0fc357
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/1b0fc357
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/1b0fc357

Branch: refs/heads/master
Commit: 1b0fc35734a07defe05d87b27db5188eb989901a
Parents: 9fb2170
Author: Mike Liddell <ml...@apache.org>
Authored: Tue May 21 13:22:27 2013 -0700
Committer: Mike Liddell <ml...@apache.org>
Committed: Tue May 21 13:23:57 2013 -0700

----------------------------------------------------------------------
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |   51 ++++++++++++++-
 1 files changed, 49 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/1b0fc357/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index fcb2ca1..7582a5e 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -509,7 +509,7 @@ public class YARNRunner implements ClientProtocol {
     }
   }
   
-  private Vertex configureReduceStage(FileSystem fs, JobID jobId,
+  private Vertex configureIntermediateReduceStage(FileSystem fs, JobID jobId,
       Configuration jobConf, String jobSubmitDir, Credentials ts,
       Map<String, LocalResource> jobLocalResources, int iReduceIndex)
       throws IOException {
@@ -538,6 +538,8 @@ public class YARNRunner implements ClientProtocol {
     vertex.setTaskLocationsHint(null);
     vertex.setTaskResource(reduceResource);
 
+    vertex.setJavaOpts(getReduceJavaOpts(conf));
+    
     return vertex;
   }
 
@@ -551,9 +553,15 @@ public class YARNRunner implements ClientProtocol {
     Vertex[] vertices = new Vertex[numIntermediateStages];
 
     for (int i = 0; i < numIntermediateStages; i++) {
-      vertices[i] = configureReduceStage(fs, jobId, jobConf, jobSubmitDir, ts,
+      vertices[i] = configureIntermediateReduceStage(fs, jobId, jobConf, jobSubmitDir, ts,
           jobLocalResources, i);
       dag.addVertex(vertices[i]);
+      
+      LOG.info("XXXX Adding intermediate vertex to DAG"
+          + ", vertexName=" + vertices[i].getVertexName()
+          + ", processor=" + vertices[i].getProcessorName()
+          + ", parrellism=" + vertices[i].getParallelism()
+          + ", javaOpts=" + vertices[i].getJavaOpts());
     }
     return vertices;
   }
@@ -604,6 +612,8 @@ public class YARNRunner implements ClientProtocol {
     mapVertex.setTaskLocationsHint(inputSplitLocations);
     mapVertex.setTaskResource(mapResource);
 
+    mapVertex.setJavaOpts(getMapJavaOpts(jobConf));
+    
     LOG.info("XXXX Adding map vertex to DAG"
         + ", vertexName=" + mapVertex.getVertexName()
         + ", processor=" + mapVertex.getProcessorName()
@@ -645,6 +655,8 @@ public class YARNRunner implements ClientProtocol {
       reduceVertex.setTaskLocationsHint(null);
       reduceVertex.setTaskResource(reduceResource);
 
+      reduceVertex.setJavaOpts(getReduceJavaOpts(jobConf));
+      
       LOG.info("XXXX Adding reduce vertex to DAG"
           + ", vertexName=" + reduceVertex.getVertexName()
           + ", processor=" + reduceVertex.getProcessorName()
@@ -1114,4 +1126,39 @@ public class YARNRunner implements ClientProtocol {
                envConf + " config settings.");
     }
   }
+  
+  private String getMapJavaOpts(Configuration jobConf) {
+    // follows pattern from YARN MapReduceChildJVM.java
+    String adminOpts = "";
+    adminOpts = jobConf.get(
+        MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,
+        MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
+    
+    String userOpts = "";
+    userOpts = 
+        jobConf.get(
+            MRJobConfig.MAP_JAVA_OPTS, // same as JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS
+                jobConf.get(
+                    JobConf.MAPRED_TASK_JAVA_OPTS,
+                    JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
+
+    return adminOpts + " " + userOpts;
+  }
+  
+  private String getReduceJavaOpts(Configuration jobConf) {
+    // follows pattern from YARN MapReduceChildJVM.java 
+    String adminOpts = "";
+    adminOpts = jobConf.get(
+        MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,
+        MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
+    
+    String userOpts = "";
+    userOpts = 
+        jobConf.get(
+            MRJobConfig.REDUCE_JAVA_OPTS, // same as JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS 
+                jobConf.get(
+                    JobConf.MAPRED_TASK_JAVA_OPTS,
+                    JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
+    return adminOpts + " " + userOpts;
+  }
 }


[29/50] [abbrv] git commit: TEZ-47. Get task max attempts from API or config (bikas)

Posted by ss...@apache.org.
TEZ-47. Get task max attempts from API or config (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/4beb98f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/4beb98f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/4beb98f3

Branch: refs/heads/master
Commit: 4beb98f3158aa2818395c805500c694b9e11c2b2
Parents: b5ebd49
Author: Bikas Saha <bi...@apache.org>
Authored: Fri May 24 12:06:05 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Fri May 24 12:06:34 2013 -0700

----------------------------------------------------------------------
 .../org/apache/tez/dag/api/TezConfiguration.java   |    5 +++++
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java      |    4 +---
 .../org/apache/tez/dag/app/dag/impl/TaskImpl.java  |    3 ++-
 3 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4beb98f3/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 3018ea1..1dec08a 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -61,6 +61,11 @@ public class TezConfiguration extends Configuration {
   public static final String DAG_MAX_TASK_FAILURES_PER_NODE = TEZ_PREFIX
       + "maxtaskfailures.per.node";
   public static final int DAG_MAX_TASK_FAILURES_PER_NODE_DEFAULT = 3;
+  
+  public static final String DAG_MAX_TASK_ATTEMPTS = 
+      DAG_AM_PREFIX + "max.task.attempts";
+  public static final int DAG_MAX_TASK_ATTEMPTS_DEFAULT = 4;
+  
   public static final String DAG_NODE_BLACKLISTING_ENABLED = TEZ_PREFIX
       + "node-blacklisting.enabled";
   public static final boolean DAG_NODE_BLACKLISTING_ENABLED_DEFAULT = true;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4beb98f3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 92ccffe..5d4242d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -101,14 +101,13 @@ public class TaskAttemptImpl implements TaskAttempt,
     EventHandler<TaskAttemptEvent> {
 
   // TODO Ensure MAPREDUCE-4457 is factored in. Also MAPREDUCE-4068.
-  // TODO Consider TAL registartion in the TaskAttempt instead of the container.
+  // TODO Consider TAL registration in the TaskAttempt instead of the container.
 
   private static final Log LOG = LogFactory.getLog(TaskAttemptImpl.class);
   private static final String LINE_SEPARATOR = System
       .getProperty("line.separator");
   
   static final TezCounters EMPTY_COUNTERS = new TezCounters();
-  private static final long MEMORY_SPLITS_RESOLUTION = 1024; //TODO Make configurable?
 
   protected final TezConfiguration conf;
   protected final int partition;
@@ -144,7 +143,6 @@ public class TaskAttemptImpl implements TaskAttempt,
   protected final String javaOpts;
   protected final boolean isRescheduled;
 
-  private boolean speculatorContainerRequestSent = false;
   protected String processorName;
   
   protected static final FailedTransitionHelper FAILED_HELPER =

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4beb98f3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 401f1b0..fb738ce 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -289,7 +289,8 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     writeLock = readWriteLock.writeLock();
     this.attempts = Collections.emptyMap();
     // TODO TEZ-47 get from conf or API
-    maxAttempts = 4;
+    maxAttempts = this.conf.getInt(TezConfiguration.DAG_MAX_TASK_ATTEMPTS, 
+                              TezConfiguration.DAG_MAX_TASK_ATTEMPTS_DEFAULT);
     taskId = new TezTaskID(vertexId, partition);
     this.partition = partition;
     this.taskAttemptListener = taskAttemptListener;


[21/50] [abbrv] git commit: TEZ-129. Set LocalDirs correctly for MapReduce tasks. (sseth)

Posted by ss...@apache.org.
TEZ-129. Set LocalDirs correctly for MapReduce tasks. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/37d7a4cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/37d7a4cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/37d7a4cc

Branch: refs/heads/master
Commit: 37d7a4cc3d65aa754c1ce59444731ec944a20381
Parents: 1b0fc35
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue May 21 16:13:00 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue May 21 16:13:21 2013 -0700

----------------------------------------------------------------------
 .../apache/tez/mapreduce/task/MRRuntimeTask.java   |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/37d7a4cc/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index 3b51889..e5dfd66 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.tez.common.TezEngineTaskContext;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
@@ -169,6 +168,7 @@ public class MRRuntimeTask extends RuntimeTask {
     }
 
     job.set(MRJobConfig.JOB_LOCAL_DIR, job.get(TezJobConfig.JOB_LOCAL_DIR));
+    job.set(MRConfig.LOCAL_DIR, job.get(TezJobConfig.LOCAL_DIRS));
 
     // setup the child's attempt directories
     // Do the task-type specific localization


[46/50] [abbrv] TEZ-175. Cleanup logs.

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
index 4c73559..a79358b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
@@ -55,12 +55,12 @@ public class AMNodeImpl implements AMNode {
   private int numSuccessfulTAs = 0;
   private boolean blacklistingEnabled;
   private boolean ignoreBlacklisting = false;
-  
+
   @SuppressWarnings("rawtypes")
   protected EventHandler eventHandler;
 
   private final List<ContainerId> containers = new LinkedList<ContainerId>();
-  
+
   //Book-keeping only. In case of Health status change.
   private final List<ContainerId> pastContainers = new LinkedList<ContainerId>();
 
@@ -69,8 +69,8 @@ public class AMNodeImpl implements AMNode {
   private final StateMachine<AMNodeState, AMNodeEventType, AMNodeEvent> stateMachine;
 
   private static StateMachineFactory
-  <AMNodeImpl, AMNodeState, AMNodeEventType, AMNodeEvent> 
-  stateMachineFactory = 
+  <AMNodeImpl, AMNodeState, AMNodeEventType, AMNodeEvent>
+  stateMachineFactory =
   new StateMachineFactory<AMNodeImpl, AMNodeState, AMNodeEventType, AMNodeEvent>(
   AMNodeState.ACTIVE)
         // Transitions from ACTIVE state.
@@ -97,7 +97,7 @@ public class AMNodeImpl implements AMNode {
     .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.UNHEALTHY, AMNodeEventType.N_TURNED_UNHEALTHY, new NodeTurnedUnhealthyTransition())
     .addTransition(AMNodeState.FORCED_ACTIVE, EnumSet.of(AMNodeState.BLACKLISTED, AMNodeState.ACTIVE), AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED, new IgnoreBlacklistingDisabledTransition())
     .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.FORCED_ACTIVE, EnumSet.of(AMNodeEventType.N_TURNED_HEALTHY, AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED), new GenericErrorTransition())
-            
+
         // Transitions from UNHEALTHY state.
     .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, AMNodeEventType.N_CONTAINER_ALLOCATED, new ContainerAllocatedWhileUnhealthyTransition())
     .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, EnumSet.of(AMNodeEventType.N_TA_SUCCEEDED, AMNodeEventType.N_TA_ENDED))
@@ -107,7 +107,7 @@ public class AMNodeImpl implements AMNode {
     .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, AMNodeEventType.N_TURNED_UNHEALTHY, new GenericErrorTransition())
 
         .installTopology();
-  
+
 
   @SuppressWarnings("rawtypes")
   public AMNodeImpl(NodeId nodeId, int maxTaskFailuresPerNode,
@@ -153,10 +153,12 @@ public class AMNodeImpl implements AMNode {
 
   @Override
   public void handle(AMNodeEvent event) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Processing AMNodeEvent " + event.getNodeId()
+          + " of type " + event.getType() + " while in state: " + getState()
+          + ". Event: " + event);
+    }
     this.writeLock.lock();
-    LOG.info("DEBUG: Processing AMNodeEvent " + event.getNodeId()
-        + " of type " + event.getType() + " while in state: " + getState()
-        + ". Event: " + event);
     try {
       final AMNodeState oldState = getState();
       try {
@@ -174,7 +176,7 @@ public class AMNodeImpl implements AMNode {
       writeLock.unlock();
     }
   }
-  
+
   protected boolean shouldBlacklistNode() {
     return blacklistingEnabled && (numFailedTAs >= maxTaskFailuresPerNode);
   }
@@ -193,7 +195,7 @@ public class AMNodeImpl implements AMNode {
   //////////////////////////////////////////////////////////////////////////////
   //                   Start of Transition Classes                            //
   //////////////////////////////////////////////////////////////////////////////
-  
+
   protected static class ContainerAllocatedTransition implements
       SingleArcTransition<AMNodeImpl, AMNodeEvent> {
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryWriter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryWriter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryWriter.java
index f683605..b3ebb8b 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryWriter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryWriter.java
@@ -35,62 +35,64 @@ import org.apache.tez.engine.common.sort.impl.IFile.Writer;
 @InterfaceStability.Unstable
 public class InMemoryWriter extends Writer {
   private static final Log LOG = LogFactory.getLog(InMemoryWriter.class);
-  
+
   private DataOutputStream out;
-  
+
   public InMemoryWriter(BoundedByteArrayOutputStream arrayStream) {
     super(null);
-    this.out = 
+    this.out =
       new DataOutputStream(new IFileOutputStream(arrayStream));
   }
-  
+
   public void append(Object key, Object value) throws IOException {
     throw new UnsupportedOperationException
     ("InMemoryWriter.append(K key, V value");
   }
-  
+
   public void append(DataInputBuffer key, DataInputBuffer value)
   throws IOException {
     int keyLength = key.getLength() - key.getPosition();
     if (keyLength < 0) {
-      throw new IOException("Negative key-length not allowed: " + keyLength + 
+      throw new IOException("Negative key-length not allowed: " + keyLength +
                             " for " + key);
     }
-    
+
     boolean sameKey = (key == IFile.REPEAT_KEY);
-    
+
     int valueLength = value.getLength() - value.getPosition();
     if (valueLength < 0) {
-      throw new IOException("Negative value-length not allowed: " + 
+      throw new IOException("Negative value-length not allowed: " +
                             valueLength + " for " + value);
     }
-    
+
     if(sameKey) {
       WritableUtils.writeVInt(out, IFile.RLE_MARKER);
       WritableUtils.writeVInt(out, valueLength);
       out.write(value.getData(), value.getPosition(), valueLength);
     } else {
-      LOG.info("XXX InMemWriter.append" + 
-          " key.data=" + key.getData() + 
-          " key.pos=" + key.getPosition() + 
-          " key.len=" +key.getLength() + 
-          " val.data=" + value.getData() + 
-          " val.pos=" + value.getPosition() + 
-          " val.len=" + value.getLength());
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("InMemWriter.append" +
+            " key.data=" + key.getData() +
+            " key.pos=" + key.getPosition() +
+            " key.len=" +key.getLength() +
+            " val.data=" + value.getData() +
+            " val.pos=" + value.getPosition() +
+            " val.len=" + value.getLength());
+      }
       WritableUtils.writeVInt(out, keyLength);
       WritableUtils.writeVInt(out, valueLength);
-      out.write(key.getData(), key.getPosition(), keyLength); 
-      out.write(value.getData(), value.getPosition(), valueLength);      
+      out.write(key.getData(), key.getPosition(), keyLength);
+      out.write(value.getData(), value.getPosition(), valueLength);
     }
-     
+
   }
 
   public void close() throws IOException {
     // Write EOF_MARKER for key/value length
     WritableUtils.writeVInt(out, IFile.EOF_MARKER);
     WritableUtils.writeVInt(out, IFile.EOF_MARKER);
-    
-    // Close the stream 
+
+    // Close the stream
     out.close();
     out = null;
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
index 76fc2e9..f5976e3 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
@@ -86,12 +86,12 @@ public abstract class ExternalSorter {
   protected SerializationFactory serializationFactory;
   protected Serializer keySerializer;
   protected Serializer valSerializer;
-  
+
   protected IndexedSorter sorter;
 
   // Compression for map-outputs
   protected CompressionCodec codec;
-  
+
   // Counters
   protected TezCounter mapOutputByteCounter;
   protected TezCounter mapOutputRecordCounter;
@@ -101,41 +101,41 @@ public abstract class ExternalSorter {
 
   public void initialize(Configuration conf, Master master)
       throws IOException, InterruptedException {
-    
+
     this.job = conf;
-    LOG.info("TEZ_ENGINE_TASK_ATTEMPT_ID: " + 
+    LOG.info("TEZ_ENGINE_TASK_ATTEMPT_ID: " +
         job.get(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID));
 
     partitions = task.getOutputSpecList().get(0).getNumOutputs();
-//    partitions = 
+//    partitions =
 //        job.getInt(
-//            TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE, 
+//            TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE,
 //            TezJobConfig.DEFAULT_TEZ_ENGINE_TASK_OUTDEGREE);
     rfs = ((LocalFileSystem)FileSystem.getLocal(job)).getRaw();
-    
+
     // sorter
     sorter = ReflectionUtils.newInstance(job.getClass(
         TezJobConfig.TEZ_ENGINE_INTERNAL_SORTER_CLASS, QuickSort.class,
         IndexedSorter.class), job);
-    
+
     comparator = ConfigUtils.getIntermediateOutputKeyComparator(job);
-    
+
     // k/v serialization
     keyClass = ConfigUtils.getIntermediateOutputKeyClass(job);
     valClass = ConfigUtils.getIntermediateOutputValueClass(job);
     serializationFactory = new SerializationFactory(job);
     keySerializer = serializationFactory.getSerializer(keyClass);
     valSerializer = serializationFactory.getSerializer(valClass);
-    
+
     //    counters
-    mapOutputByteCounter = 
+    mapOutputByteCounter =
         runningTaskContext.getTaskReporter().getCounter(TaskCounter.MAP_OUTPUT_BYTES);
     mapOutputRecordCounter =
       runningTaskContext.getTaskReporter().getCounter(TaskCounter.MAP_OUTPUT_RECORDS);
-    fileOutputByteCounter = 
+    fileOutputByteCounter =
         runningTaskContext.getTaskReporter().
             getCounter(TaskCounter.MAP_OUTPUT_MATERIALIZED_BYTES);
-    spilledRecordsCounter = 
+    spilledRecordsCounter =
         runningTaskContext.getTaskReporter().getCounter(TaskCounter.SPILLED_RECORDS);
     // compression
     if (ConfigUtils.shouldCompressIntermediateOutput(job)) {
@@ -146,14 +146,13 @@ public abstract class ExternalSorter {
       codec = null;
     }
 
-    // Task outputs 
+    // Task outputs
     mapOutputFile =
         (TezTaskOutput) ReflectionUtils.newInstance(
             conf.getClass(
-                Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER, 
+                Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
                 TezTaskOutputFiles.class), conf);
-//    LOG.info("XXX mapOutputFile: " + mapOutputFile.getClass());
-    
+
     // sortPhase
     sortPhase  = runningTaskContext.getProgress().addPhase("sort", 0.333f);
   }
@@ -227,13 +226,8 @@ public abstract class ExternalSorter {
     if (!src.renameTo(dst)) {
       throw new IOException("Unable to rename " + src + " to " + dst);
     }
-//    LOG.info("XXX sameVolRename src=" + src + ", dst=" + dst);
   }
 
-//  public ExternalSorter() {
-//    super();
-//  }
-  
   public ExternalSorter(TezEngineTaskContext tezEngineTask) {
     this.task = tezEngineTask;
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/IFileOutputStream.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/IFileOutputStream.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/IFileOutputStream.java
index 75fcd68..00b8958 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/IFileOutputStream.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/IFileOutputStream.java
@@ -36,11 +36,9 @@ import org.apache.hadoop.util.DataChecksum;
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class IFileOutputStream extends FilterOutputStream {
-  
-  private static final Log LOG = LogFactory.getLog(IFileOutputStream.class);
-  
+
   /**
-   * The output stream to be checksummed. 
+   * The output stream to be checksummed.
    */
   private final DataChecksum sum;
   private byte[] barray;
@@ -62,7 +60,7 @@ public class IFileOutputStream extends FilterOutputStream {
     buffer = new byte[4096];
     offset = 0;
   }
-  
+
   @Override
   public void close() throws IOException {
     if (closed) {
@@ -105,8 +103,8 @@ public class IFileOutputStream extends FilterOutputStream {
     // FIXME if needed re-enable this in debug mode
     if (LOG.isDebugEnabled()) {
       LOG.debug("XXX checksum" +
-          " b=" + b + " off=" + off + 
-          " buffer=" + " offset=" + offset + 
+          " b=" + b + " off=" + off +
+          " buffer=" + " offset=" + offset +
           " len=" + len);
     }
     */
@@ -123,7 +121,7 @@ public class IFileOutputStream extends FilterOutputStream {
     checksum(b, off, len);
     out.write(b,off,len);
   }
- 
+
   @Override
   public void write(int b) throws IOException {
     barray[0] = (byte) (b & 0xFF);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
index e567492..adbff22 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
@@ -57,16 +57,16 @@ import org.apache.tez.engine.records.OutputContext;
 
 @SuppressWarnings({"unchecked", "rawtypes"})
 public class DefaultSorter extends ExternalSorter implements IndexedSortable {
-  
+
   private static final Log LOG = LogFactory.getLog(DefaultSorter.class);
-  
+
   /**
    * The size of each record in the index file for the map-outputs.
    */
   public static final int MAP_OUTPUT_INDEX_RECORD_LENGTH = 24;
 
   private final static int APPROX_HEADER_LENGTH = 150;
-  
+
   // k/v accounting
   IntBuffer kvmeta; // metadata overlay on backing store
   int kvstart;            // marks origin of spill metadata
@@ -118,15 +118,15 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
   }
 
   @Override
-  public void initialize(Configuration conf, Master master) 
+  public void initialize(Configuration conf, Master master)
       throws IOException, InterruptedException {
     if (task == null) {
       LOG.info("Bailing!", new IOException());
       return;
     }
-    
+
     super.initialize(conf, master);
-    
+
     // sanity checks
     final float spillper = job.getFloat(
         TezJobConfig.TEZ_ENGINE_SORT_SPILL_PERCENT,
@@ -144,7 +144,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
 
     indexCacheMemoryLimit = job.getInt(TezJobConfig.TEZ_ENGINE_INDEX_CACHE_MEMORY_LIMIT_BYTES,
                                        TezJobConfig.DEFAULT_TEZ_ENGINE_INDEX_CACHE_MEMORY_LIMIT_BYTES);
-    
+
     // buffers and accounting
     int maxMemUsage = sortmb << 20;
     maxMemUsage -= maxMemUsage % METASIZE;
@@ -169,8 +169,8 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
 
     // k/v serialization
     valSerializer.open(bb);
-    keySerializer.open(bb);   
-    
+    keySerializer.open(bb);
+
     spillInProgress = false;
     minSpillsForCombine = job.getInt(TezJobConfig.TEZ_ENGINE_COMBINE_MIN_SPILLS, 3);
     spillThread.setDaemon(true);
@@ -193,7 +193,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
   }
 
   @Override
-  public void write(Object key, Object value) 
+  public void write(Object key, Object value)
       throws IOException, InterruptedException {
     collect(
         key, value, partitioner.getPartition(key, value, partitions));
@@ -673,7 +673,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
             spillLock.unlock();
             sortAndSpill();
           } catch (Throwable t) {
-            LOG.warn("ZZZZ: Got an exception in sortAndSpill", t);
+            LOG.warn("Got an exception in sortAndSpill", t);
             sortSpillException = t;
           } finally {
             spillLock.lock();
@@ -727,23 +727,23 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
   int getMetaStart() {
     return kvend / NMETA;
   }
-  
+
   int getMetaEnd() {
     return 1 + // kvend is a valid record
         (kvstart >= kvend
         ? kvstart
         : kvmeta.capacity() + kvstart) / NMETA;
   }
-  
-  protected void sortAndSpill() 
+
+  protected void sortAndSpill()
       throws IOException, InterruptedException {
     final int mstart = getMetaStart();
     final int mend = getMetaEnd();
     sorter.sort(this, mstart, mend, runningTaskContext.getTaskReporter());
-    spill(mstart, mend); 
+    spill(mstart, mend);
   }
-  
-  protected void spill(int mstart, int mend) 
+
+  protected void spill(int mstart, int mend)
       throws IOException, InterruptedException {
 
     //approximate the length of the output file to be the length of the
@@ -775,7 +775,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
                 kvmeta.get(offsetFor(spindex) + PARTITION) == i) {
               final int kvoff = offsetFor(spindex);
               key.reset(
-                  kvbuffer, 
+                  kvbuffer,
                   kvmeta.get(kvoff + KEYSTART),
                   (kvmeta.get(kvoff + VALSTART) - kvmeta.get(kvoff + KEYSTART))
                   );
@@ -795,7 +795,9 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
             if (spstart != spindex) {
               TezRawKeyValueIterator kvIter =
                 new MRResultIterator(spstart, spindex);
-              LOG.info("DEBUG: Running combine processor");
+              if (LOG.isDebugEnabled()) {
+                LOG.debug("Running combine processor");
+              }
               runCombineProcessor(kvIter, writer);
             }
           }
@@ -804,10 +806,10 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
           writer.close();
 
           // record offsets
-          final TezIndexRecord rec = 
+          final TezIndexRecord rec =
               new TezIndexRecord(
-                  segmentStart, 
-                  writer.getRawLength(), 
+                  segmentStart,
+                  writer.getRawLength(),
                   writer.getCompressedLength());
           spillRec.putIndex(rec, i);
 
@@ -870,10 +872,10 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
           writer.close();
 
           // record offsets
-          TezIndexRecord rec = 
+          TezIndexRecord rec =
               new TezIndexRecord(
-                  segmentStart, 
-                  writer.getRawLength(), 
+                  segmentStart,
+                  writer.getRawLength(),
                   writer.getCompressedLength());
           spillRec.putIndex(rec, i);
 
@@ -913,7 +915,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
       : (bufvoid - kvmeta.get(kvoff + VALSTART)) + nextindex;
       return vallen;
   }
-  
+
   /**
    * Given an offset, populate vbytes with the associated set of
    * deserialized value bytes. Should only be called during a spill.
@@ -936,7 +938,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
     public InMemValBytes(int bufvoid) {
       this.bufvoid = bufvoid;
     }
-    
+
     public void reset(byte[] buffer, int start, int length) {
       this.buffer = buffer;
       this.start = start;
@@ -957,7 +959,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
   InMemValBytes createInMemValBytes() {
     return new InMemValBytes(bufvoid);
   }
-  
+
   protected class MRResultIterator implements TezRawKeyValueIterator {
     private final DataInputBuffer keybuf = new DataInputBuffer();
     private final InMemValBytes vbytes = createInMemValBytes();
@@ -1031,7 +1033,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
 
     if (numSpills == 0) {
       //create dummy files
-      
+
       TezSpillRecord sr = new TezSpillRecord(partitions);
       try {
         for (int i = 0; i < partitions; i++) {
@@ -1039,11 +1041,11 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
           Writer writer =
             new Writer(job, finalOut, keyClass, valClass, codec, null);
           writer.close();
-          
-          TezIndexRecord rec = 
+
+          TezIndexRecord rec =
               new TezIndexRecord(
-                  segmentStart, 
-                  writer.getRawLength(), 
+                  segmentStart,
+                  writer.getRawLength(),
                   writer.getCompressedLength());
           sr.putIndex(rec, i);
         }
@@ -1057,7 +1059,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
     else {
       sortPhase.addPhases(partitions); // Divide sort phase into sub-phases
       TezMerger.considerFinalMergeForProgress();
-      
+
       final TezSpillRecord spillRec = new TezSpillRecord(partitions);
       for (int parts = 0; parts < partitions; parts++) {
         //create the segments to be merged
@@ -1074,13 +1076,13 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
           if (LOG.isDebugEnabled()) {
             LOG.debug("MapId=" + mapId + " Reducer=" + parts +
                 "Spill =" + i + "(" + indexRecord.getStartOffset() + "," +
-                indexRecord.getRawLength() + ", " + 
+                indexRecord.getRawLength() + ", " +
                 indexRecord.getPartLength() + ")");
           }
         }
 
-        int mergeFactor = 
-            job.getInt(TezJobConfig.TEZ_ENGINE_IO_SORT_FACTOR, 
+        int mergeFactor =
+            job.getInt(TezJobConfig.TEZ_ENGINE_IO_SORT_FACTOR,
                 TezJobConfig.DEFAULT_TEZ_ENGINE_IO_SORT_FACTOR);
         // sort the segments only if there are intermediate merges
         boolean sortSegments = segmentList.size() > mergeFactor;
@@ -1089,9 +1091,9 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
                        keyClass, valClass, codec,
                        segmentList, mergeFactor,
                        new Path(mapId.toString()),
-                       (RawComparator)ConfigUtils.getIntermediateOutputKeyComparator(job), 
+                       (RawComparator)ConfigUtils.getIntermediateOutputKeyComparator(job),
                        runningTaskContext.getTaskReporter(), sortSegments,
-                       null, spilledRecordsCounter, 
+                       null, spilledRecordsCounter,
                        sortPhase.phase());
 
         //write merged output to disk
@@ -1109,10 +1111,10 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
 
         sortPhase.startNextPhase();
         // record offsets
-        final TezIndexRecord rec = 
+        final TezIndexRecord rec =
             new TezIndexRecord(
-                segmentStart, 
-                writer.getRawLength(), 
+                segmentStart,
+                writer.getRawLength(),
                 writer.getCompressedLength());
         spillRec.putIndex(rec, parts);
       }
@@ -1123,7 +1125,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
       }
     }
   }
-  
+
   @Override
   public OutputContext getOutputContext() {
     return null;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
index 5fb6519..b7874f0 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
@@ -55,20 +55,22 @@ public class TezTaskOutputFiles extends TezTaskOutput {
   }
 
   // assume configured to $localdir/usercache/$user/appcache/$appId
-  private LocalDirAllocator lDirAlloc = 
+  private LocalDirAllocator lDirAlloc =
     new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
 
   private Path getAttemptOutputDir() {
-    LOG.info("DEBUG: getAttemptOutputDir: "
-        + Constants.TASK_OUTPUT_DIR + "/"
-        + conf.get(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID));
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("getAttemptOutputDir: "
+          + Constants.TASK_OUTPUT_DIR + "/"
+          + conf.get(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID));
+    }
     return new Path(Constants.TASK_OUTPUT_DIR,
         conf.get(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID));
   }
-  
+
   /**
    * Return the path to local map output file created earlier
-   * 
+   *
    * @return path
    * @throws IOException
    */
@@ -80,13 +82,13 @@ public class TezTaskOutputFiles extends TezTaskOutput {
 
   /**
    * Create a local map output file name.
-   * 
+   *
    * @param size the size of the file
    * @return path
    * @throws IOException
    */
   public Path getOutputFileForWrite(long size) throws IOException {
-    Path attemptOutput = 
+    Path attemptOutput =
       new Path(getAttemptOutputDir(), Constants.TEZ_ENGINE_TASK_OUTPUT_FILENAME_STRING);
     return lDirAlloc.getLocalPathForWrite(attemptOutput.toString(), size, conf);
   }
@@ -103,7 +105,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
 
   /**
    * Return the path to a local map output index file created earlier
-   * 
+   *
    * @return path
    * @throws IOException
    */
@@ -116,7 +118,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
 
   /**
    * Create a local map output index file name.
-   * 
+   *
    * @param size the size of the file
    * @return path
    * @throws IOException
@@ -142,7 +144,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
 
   /**
    * Return a local map spill file created earlier.
-   * 
+   *
    * @param spillNumber the number
    * @return path
    * @throws IOException
@@ -155,7 +157,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
 
   /**
    * Create a local map spill file name.
-   * 
+   *
    * @param spillNumber the number
    * @param size the size of the file
    * @return path
@@ -170,7 +172,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
 
   /**
    * Return a local map spill index file created earlier
-   * 
+   *
    * @param spillNumber the number
    * @return path
    * @throws IOException
@@ -183,7 +185,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
 
   /**
    * Create a local map spill index file name.
-   * 
+   *
    * @param spillNumber the number
    * @param size the size of the file
    * @return path
@@ -198,10 +200,10 @@ public class TezTaskOutputFiles extends TezTaskOutput {
 
   /**
    * Return a local reduce input file created earlier
-   * 
+   *
    * @param mapId a map task id
    * @return path
-   * @throws IOException 
+   * @throws IOException
    */
   public Path getInputFile(int mapId) throws IOException {
     throw new UnsupportedOperationException("Incompatible with LocalRunner");
@@ -209,7 +211,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
 
   /**
    * Create a local reduce input file name.
-   * 
+   *
    * @param mapId a map task id
    * @param size the size of the file
    * @return path
@@ -235,5 +237,5 @@ public class TezTaskOutputFiles extends TezTaskOutput {
   public Configuration getConf() {
     return conf;
   }
-  
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java
index 1d2360b..a954f6e 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java
@@ -30,28 +30,28 @@ import org.apache.tez.engine.common.task.local.output.TezTaskOutput;
 public class LocalOnFileSorterOutput extends OnFileSortedOutput {
 
   private static final Log LOG = LogFactory.getLog(LocalOnFileSorterOutput.class);
-  
+
   public LocalOnFileSorterOutput(TezEngineTaskContext task) throws IOException {
     super(task);
   }
 
   @Override
   public void close() throws IOException, InterruptedException {
-    LOG.info("XXX close");
-
+    LOG.debug("Closing LocalOnFileSorterOutput");
     super.close();
 
-
     TezTaskOutput mapOutputFile = sorter.getMapOutput();
     FileSystem localFs = FileSystem.getLocal(mapOutputFile.getConf());
 
     Path src = mapOutputFile.getOutputFile();
-    Path dst = 
+    Path dst =
         mapOutputFile.getInputFileForWrite(
             sorter.getTaskAttemptId().getTaskID(),
             localFs.getFileStatus(src).getLen());
 
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Renaming src = " + src + ", dst = " + dst);
+    }
     localFs.rename(src, dst);
-    LOG.info("XXX renaming src = " + src + ", dst = " + dst);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
index 900c2f0..1946a62 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
@@ -62,7 +62,7 @@ public class RuntimeUtils {
 
   public static RuntimeTask createRuntimeTask(
       TezEngineTaskContext taskContext) {
-    LOG.info("TaskContext"
+    LOG.info("Creating a runtime task from TaskContext"
         + ", Processor: " + taskContext.getProcessorName()
         + ", InputCount=" + taskContext.getInputSpecList().size()
         + ", OutputCount=" + taskContext.getOutputSpecList().size());
@@ -78,38 +78,45 @@ public class RuntimeUtils {
       Input[] inputs;
       Output[] outputs;
       if (taskContext.getInputSpecList().isEmpty()) {
-        LOG.info("Initializing task with 0 inputs");
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Initializing task with 0 inputs");
+        }
         inputs = new Input[0];
       } else {
         int iSpecCount = taskContext.getInputSpecList().size();
         inputs = new Input[iSpecCount];
         for (int i = 0; i < iSpecCount; ++i) {
           InputSpec inSpec = taskContext.getInputSpecList().get(i);
-          LOG.info("XXXX Using Input"
-              + ", index=" + i
-              + ", inputClass=" + inSpec.getInputClassName());
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Using Input"
+                + ", index=" + i
+                + ", inputClass=" + inSpec.getInputClassName());
+          }
           Class<?> inputClazz = Class.forName(inSpec.getInputClassName());
           Input input = (Input) getNewInstance(inputClazz, taskContext);
           inputs[i] = input;
         }
       }
       if (taskContext.getOutputSpecList().isEmpty()) {
-        LOG.info("Initializing task with 0 outputs");
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Initializing task with 0 outputs");
+        }
         outputs = new Output[0];
       } else {
         int oSpecCount = taskContext.getOutputSpecList().size();
         outputs = new Output[oSpecCount];
         for (int i = 0; i < oSpecCount; ++i) {
           OutputSpec outSpec = taskContext.getOutputSpecList().get(i);
-          LOG.info("XXXX Using Output"
-              + ", index=" + i
-              + ", output=" + outSpec.getOutputClassName());
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Using Output"
+                + ", index=" + i
+                + ", output=" + outSpec.getOutputClassName());
+          }
           Class<?> outputClazz = Class.forName(outSpec.getOutputClassName());
           Output output = (Output) getNewInstance(outputClazz, taskContext);
           outputs[i] = output;
         }
       }
-      // t = new RuntimeTask(taskContext, processor, inputs, outputs);
       t = createRuntime(taskContext, processor, inputs, outputs);
     } catch (ClassNotFoundException e) {
       throw new YarnException("Unable to initialize RuntimeTask, context="
@@ -117,7 +124,7 @@ public class RuntimeUtils {
     }
     return t;
   }
-  
+
   private static RuntimeTask createRuntime(TezEngineTaskContext taskContext,
       Processor processor, Input[] inputs, Output[] outputs) {
     try {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 5dade2b..e930044 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -106,7 +106,7 @@ public class LocalJobRunner implements ClientProtocol {
   private AtomicInteger map_tasks = new AtomicInteger(0);
   private int reduce_tasks = 0;
   final Random rand = new Random();
-  
+
   private LocalJobRunnerMetrics myMetrics = null;
 
   private static final String jobDir =  "localRunner/";
@@ -129,7 +129,7 @@ public class LocalJobRunner implements ClientProtocol {
     // This is analogous to JobTracker's system directory.
     private Path systemJobDir;
     private Path systemJobFile;
-    
+
     // The job directory for the task.  Analagous to a task's job directory.
     private Path localJobDir;
     private Path localJobFile;
@@ -149,13 +149,13 @@ public class LocalJobRunner implements ClientProtocol {
     private JobProfile profile;
     private FileSystem localFs;
     boolean killed = false;
-    
+
     private LocalDistributedCacheManager localDistributedCacheManager;
 
     public long getProtocolVersion(String protocol, long clientVersion) {
       return TaskUmbilicalProtocol.versionID;
     }
-    
+
     @Override
     public ProtocolSignature getProtocolSignature(String protocol,
         long clientVersion, int clientMethodsHash) throws IOException {
@@ -176,7 +176,7 @@ public class LocalJobRunner implements ClientProtocol {
       // this will trigger localFile to be re-written again.
       localDistributedCacheManager = new LocalDistributedCacheManager();
       localDistributedCacheManager.setup(conf);
-      
+
       // Write out configuration file.  Instead of copying it from
       // systemJobFile, we re-write it, since setup(), above, may have
       // updated it.
@@ -193,11 +193,11 @@ public class LocalJobRunner implements ClientProtocol {
         setContextClassLoader(localDistributedCacheManager.makeClassLoader(
                 getContextClassLoader()));
       }
-      
-      profile = new JobProfile(job.getUser(), id, systemJobFile.toString(), 
+
+      profile = new JobProfile(job.getUser(), id, systemJobFile.toString(),
                                "http://localhost:8080/", job.getJobName());
-      status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING, 
-          profile.getUser(), profile.getJobName(), profile.getJobFile(), 
+      status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING,
+          profile.getUser(), profile.getJobName(), profile.getJobFile(),
           profile.getURL().toString());
 
       jobs.put(id, this);
@@ -235,7 +235,7 @@ public class LocalJobRunner implements ClientProtocol {
           TaskAttemptID mapId = new TaskAttemptID(new TaskID(
               jobId, TaskType.MAP, taskId), 0);
           LOG.info("Starting task: " + mapId);
-          final String user = 
+          final String user =
               UserGroupInformation.getCurrentUser().getShortUserName();
           setupChildMapredLocalDirs(mapId, user, localConf);
           localConf.setUser(user);
@@ -244,7 +244,7 @@ public class LocalJobRunner implements ClientProtocol {
               IDConverter.fromMRTaskAttemptId(mapId);
           mapIds.add(mapId);
           // FIXME invalid task context
-          TezEngineTaskContext taskContext = 
+          TezEngineTaskContext taskContext =
               new TezEngineTaskContext(
                   tezMapId, user, localConf.getJobName(), "TODO_vertexName",
                   MapProcessor.class.getName(),
@@ -346,7 +346,7 @@ public class LocalJobRunner implements ClientProtocol {
       return executor;
     }
 
-    private org.apache.hadoop.mapreduce.OutputCommitter 
+    private org.apache.hadoop.mapreduce.OutputCommitter
     createOutputCommitter(boolean newApiCommitter, JobID jobId, Configuration conf) throws Exception {
       org.apache.hadoop.mapreduce.OutputCommitter committer = null;
 
@@ -358,7 +358,7 @@ public class LocalJobRunner implements ClientProtocol {
             new org.apache.hadoop.mapreduce.TaskID(jobId, TaskType.MAP, 0);
         org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID =
             new org.apache.hadoop.mapreduce.TaskAttemptID(taskId, 0);
-        org.apache.hadoop.mapreduce.TaskAttemptContext taskContext = 
+        org.apache.hadoop.mapreduce.TaskAttemptContext taskContext =
             new TaskAttemptContextImpl(conf, taskAttemptID);
         @SuppressWarnings("rawtypes")
         OutputFormat outputFormat =
@@ -377,7 +377,7 @@ public class LocalJobRunner implements ClientProtocol {
     public void run() {
       JobID jobId = profile.getJobID();
       JobContext jContext = new JobContextImpl(job, jobId);
-      
+
       org.apache.hadoop.mapreduce.OutputCommitter outputCommitter = null;
       try {
         outputCommitter = createOutputCommitter(conf.getUseNewMapper(), jobId, conf);
@@ -385,9 +385,9 @@ public class LocalJobRunner implements ClientProtocol {
         LOG.info("Failed to createOutputCommitter", e);
         return;
       }
-      
+
       try {
-        TaskSplitMetaInfo[] taskSplitMetaInfos = 
+        TaskSplitMetaInfo[] taskSplitMetaInfos =
           SplitMetaInfoReader.readSplitMetaInfo(jobId, localFs, conf, systemJobDir);
 
         int numReduceTasks = job.getNumReduceTasks();
@@ -440,7 +440,7 @@ public class LocalJobRunner implements ClientProtocol {
         LOG.info("Starting task: " + reduceId);
         try {
           if (numReduceTasks > 0) {
-            String user = 
+            String user =
                 UserGroupInformation.getCurrentUser().getShortUserName();
             JobConf localConf = new JobConf(job);
             localConf.setUser(user);
@@ -457,14 +457,16 @@ public class LocalJobRunner implements ClientProtocol {
                 Collections.singletonList(new OutputSpec("TODO_targetVertex",
                     0, SimpleOutput.class.getName())));
 
-            // move map output to reduce input  
+            // move map output to reduce input
             for (int i = 0; i < mapIds.size(); i++) {
               if (!this.isInterrupted()) {
                 TaskAttemptID mapId = mapIds.get(i);
-                LOG.info("XXX mapId: " + i + 
-                    " LOCAL_DIR = " + 
-                    mapOutputFiles.get(mapId).getConf().get(
-                        TezJobConfig.LOCAL_DIRS));
+                if (LOG.isDebugEnabled()) {
+                  LOG.debug("XXX mapId: " + i +
+                      " LOCAL_DIR = " +
+                      mapOutputFiles.get(mapId).getConf().get(
+                          TezJobConfig.LOCAL_DIRS));
+                }
                 Path mapOut = mapOutputFiles.get(mapId).getOutputFile();
                 TezTaskOutput localOutputFile = new TezLocalTaskOutputFiles();
                 localOutputFile.setConf(localConf);
@@ -514,7 +516,7 @@ public class LocalJobRunner implements ClientProtocol {
 
       } catch (Throwable t) {
         try {
-          outputCommitter.abortJob(jContext, 
+          outputCommitter.abortJob(jContext,
             org.apache.hadoop.mapreduce.JobStatus.State.FAILED);
         } catch (IOException ioe) {
           LOG.info("Error cleaning up job:" + id);
@@ -547,7 +549,7 @@ public class LocalJobRunner implements ClientProtocol {
         throws IOException {
       return null;
     }
-    
+
     @Override
     public synchronized boolean statusUpdate(TezTaskAttemptID taskId,
         TezTaskStatus taskStatus) throws IOException, InterruptedException {
@@ -596,7 +598,7 @@ public class LocalJobRunner implements ClientProtocol {
      */
     @Override
     public void commitPending(TezTaskAttemptID taskid,
-                              TezTaskStatus taskStatus) 
+                              TezTaskStatus taskStatus)
     throws IOException, InterruptedException {
       statusUpdate(taskid, taskStatus);
     }
@@ -605,17 +607,17 @@ public class LocalJobRunner implements ClientProtocol {
     public void reportDiagnosticInfo(TezTaskAttemptID taskid, String trace) {
       // Ignore for now
     }
-    
+
     @Override
     public boolean ping(TezTaskAttemptID taskid) throws IOException {
       return true;
     }
-    
+
     @Override
     public boolean canCommit(TezTaskAttemptID taskid) throws IOException {
       return true;
     }
-    
+
     @Override
     public void done(TezTaskAttemptID taskId) throws IOException {
       int taskIndex = mapIds.indexOf(taskId);
@@ -627,25 +629,25 @@ public class LocalJobRunner implements ClientProtocol {
     }
 
     @Override
-    public synchronized void fsError(TezTaskAttemptID taskId, String message) 
+    public synchronized void fsError(TezTaskAttemptID taskId, String message)
     throws IOException {
       LOG.fatal("FSError: "+ message + "from task: " + taskId);
     }
 
     @Override
-    public void shuffleError(TezTaskAttemptID taskId, String message) 
+    public void shuffleError(TezTaskAttemptID taskId, String message)
         throws IOException {
       LOG.fatal("shuffleError: "+ message + "from task: " + taskId);
     }
-    
+
     @Override
-    public synchronized void fatalError(TezTaskAttemptID taskId, String msg) 
+    public synchronized void fatalError(TezTaskAttemptID taskId, String msg)
     throws IOException {
       LOG.fatal("Fatal: "+ msg + "from task: " + taskId);
     }
-    
+
     @Override
-    public TezTaskDependencyCompletionEventsUpdate 
+    public TezTaskDependencyCompletionEventsUpdate
     getDependentTasksCompletionEvents(
         int fromEventIdx, int maxEventsToFetch,
         TezTaskAttemptID reduce) {
@@ -704,7 +706,7 @@ public class LocalJobRunner implements ClientProtocol {
     throw new UnsupportedOperationException("Changing job priority " +
                       "in LocalJobRunner is not supported.");
   }
-  
+
   /** Throws {@link UnsupportedOperationException} */
   public boolean killTask(org.apache.hadoop.mapreduce.TaskAttemptID taskId,
       boolean shouldFail) throws IOException {
@@ -722,10 +724,10 @@ public class LocalJobRunner implements ClientProtocol {
     Job job = jobs.get(JobID.downgrade(id));
     if(job != null)
       return job.status;
-    else 
+    else
       return null;
   }
-  
+
   public org.apache.hadoop.mapreduce.Counters getJobCounters(
       org.apache.hadoop.mapreduce.JobID id) {
     Job job = jobs.get(JobID.downgrade(id));
@@ -737,7 +739,7 @@ public class LocalJobRunner implements ClientProtocol {
   public String getFilesystemName() throws IOException {
     return fs.getUri().toString();
   }
-  
+
   public ClusterMetrics getClusterMetrics() {
     int numMapTasks = map_tasks.get();
     return new ClusterMetrics(numMapTasks, reduce_tasks, numMapTasks,
@@ -748,25 +750,25 @@ public class LocalJobRunner implements ClientProtocol {
     return JobTrackerStatus.RUNNING;
   }
 
-  public long getTaskTrackerExpiryInterval() 
+  public long getTaskTrackerExpiryInterval()
       throws IOException, InterruptedException {
     return 0;
   }
 
-  /** 
-   * Get all active trackers in cluster. 
+  /**
+   * Get all active trackers in cluster.
    * @return array of TaskTrackerInfo
    */
-  public TaskTrackerInfo[] getActiveTrackers() 
+  public TaskTrackerInfo[] getActiveTrackers()
       throws IOException, InterruptedException {
     return null;
   }
 
-  /** 
-   * Get all blacklisted trackers in cluster. 
+  /**
+   * Get all blacklisted trackers in cluster.
    * @return array of TaskTrackerInfo
    */
-  public TaskTrackerInfo[] getBlacklistedTrackers() 
+  public TaskTrackerInfo[] getBlacklistedTrackers()
       throws IOException, InterruptedException {
     return null;
   }
@@ -776,10 +778,10 @@ public class LocalJobRunner implements ClientProtocol {
       , int fromEventId, int maxEvents) throws IOException {
     return TaskCompletionEvent.EMPTY_ARRAY;
   }
-  
+
   public org.apache.hadoop.mapreduce.JobStatus[] getAllJobs() {return null;}
 
-  
+
   /**
    * Returns the diagnostic information for a particular task in the given job.
    * To be implemented
@@ -794,7 +796,7 @@ public class LocalJobRunner implements ClientProtocol {
    */
   public String getSystemDir() {
     Path sysDir = new Path(
-      conf.get(JTConfig.JT_SYSTEM_DIR, "/tmp/hadoop/mapred/system"));  
+      conf.get(JTConfig.JT_SYSTEM_DIR, "/tmp/hadoop/mapred/system"));
     return fs.makeQualified(sysDir).toString();
   }
 
@@ -809,7 +811,7 @@ public class LocalJobRunner implements ClientProtocol {
    * @see org.apache.hadoop.mapreduce.protocol.ClientProtocol#getStagingAreaDir()
    */
   public String getStagingAreaDir() throws IOException {
-    Path stagingRootDir = new Path(conf.get(JTConfig.JT_STAGING_AREA_ROOT, 
+    Path stagingRootDir = new Path(conf.get(JTConfig.JT_STAGING_AREA_ROOT,
         "/tmp/hadoop/mapred/staging"));
     UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
     String user;
@@ -820,7 +822,7 @@ public class LocalJobRunner implements ClientProtocol {
     }
     return fs.makeQualified(new Path(stagingRootDir, user+"/.staging")).toString();
   }
-  
+
   public String getJobHistoryDir() {
     return null;
   }
@@ -847,7 +849,7 @@ public class LocalJobRunner implements ClientProtocol {
   }
 
   @Override
-  public org.apache.hadoop.mapreduce.QueueAclsInfo[] 
+  public org.apache.hadoop.mapreduce.QueueAclsInfo[]
       getQueueAclsForCurrentUser() throws IOException{
     return null;
   }
@@ -879,7 +881,7 @@ public class LocalJobRunner implements ClientProtocol {
   }
 
   @Override
-  public Token<DelegationTokenIdentifier> 
+  public Token<DelegationTokenIdentifier>
      getDelegationToken(Text renewer) throws IOException, InterruptedException {
     return null;
   }
@@ -896,10 +898,10 @@ public class LocalJobRunner implements ClientProtocol {
       throws IOException, InterruptedException {
     throw new UnsupportedOperationException("Not supported");
   }
-  
+
   static void setupChildMapredLocalDirs(
       TaskAttemptID taskAttemptID, String user, JobConf conf) {
-    String[] localDirs = 
+    String[] localDirs =
         conf.getTrimmedStrings(
             TezJobConfig.LOCAL_DIRS, TezJobConfig.DEFAULT_LOCAL_DIRS);
     String jobId = taskAttemptID.getJobID().toString();
@@ -912,17 +914,17 @@ public class LocalJobRunner implements ClientProtocol {
       childMapredLocalDir.append("," + localDirs[i] + Path.SEPARATOR
           + getLocalTaskDir(user, jobId, taskId, isCleanup));
     }
-    LOG.info(TezJobConfig.LOCAL_DIRS + " for child : " + taskAttemptID + 
+    LOG.info(TezJobConfig.LOCAL_DIRS + " for child : " + taskAttemptID +
         " is " + childMapredLocalDir);
     conf.set(TezJobConfig.LOCAL_DIRS, childMapredLocalDir.toString());
-    conf.setClass(Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER, 
+    conf.setClass(Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
         TezLocalTaskOutputFiles.class, TezTaskOutput.class);
   }
-  
+
   static final String TASK_CLEANUP_SUFFIX = ".cleanup";
   static final String SUBDIR = jobDir;
   static final String JOBCACHE = "jobcache";
-  
+
   static String getLocalTaskDir(String user, String jobid, String taskid,
       boolean isCleanupAttempt) {
     String taskDir = SUBDIR + Path.SEPARATOR + user + Path.SEPARATOR + JOBCACHE
@@ -932,6 +934,6 @@ public class LocalJobRunner implements ClientProtocol {
     }
     return taskDir;
   }
-  
-  
+
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
index 72246c0..0fcfe65 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
@@ -61,12 +61,14 @@ public class SplitMetaInfoReaderTez {
         conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR),
         MRJobConfig.JOB_SPLIT_METAINFO);
     String jobSplitFile = MRJobConfig.JOB_SPLIT;
-    
+
     File file = new File(metaSplitFile.toUri().getPath()).getAbsoluteFile();
-    LOG.info("DEBUG: Setting up JobSplitIndex with JobSplitFile at: "
-        + file.getAbsolutePath() + ", defaultFS from conf: "
-        + FileSystem.getDefaultUri(conf));
-    
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Setting up JobSplitIndex with JobSplitFile at: "
+          + file.getAbsolutePath() + ", defaultFS from conf: "
+          + FileSystem.getDefaultUri(conf));
+    }
+
     FileStatus fStatus = fs.getFileStatus(metaSplitFile);
     if (maxMetaInfoSize > 0 && fStatus.getLen() > maxMetaInfoSize) {
       throw new IOException("Split metadata size exceeded " + maxMetaInfoSize

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
index b8a99ee..a3880f4 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
@@ -83,8 +83,9 @@ public class MRCombiner implements Processor {
   @Override
   public void process(Input[] in, Output[] out) throws IOException,
       InterruptedException {
-    LOG.info("DEBUG: Running MRCombiner"
-        + ", usingNewAPI=" + useNewApi);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Running MRCombiner, usingNewAPI=" + useNewApi);
+    }
 
     CombineInput input = (CombineInput)in[0];
     CombineOutput output = (CombineOutput)out[0];
@@ -334,8 +335,6 @@ public class MRCombiner implements Processor {
           comparator,
           keyClass,
           valueClass);
-    LOG.info("DEBUG: Using combineKeyClass: "
-          + keyClass + ", combineValueClass: " + valueClass);
 
     org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE>.Context
         reducerContext = new

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
index 9f1ae88..54648f5 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
@@ -31,7 +31,7 @@ import org.apache.tez.mapreduce.hadoop.DeprecatedKeys.MultiStageKeys;
 public class MultiStageMRConfToTezTranslator {
 
   private static final Log LOG = LogFactory.getLog(MultiStageMRConfToTezTranslator.class);
-  
+
   private enum DeprecationReason {
     DEPRECATED_DIRECT_TRANSLATION, DEPRECATED_MULTI_STAGE
   }
@@ -55,14 +55,14 @@ public class MultiStageMRConfToTezTranslator {
     int numEdges = totalStages - 1;
 
     Configuration[] allConfs = extractStageConfs(newConf, numEdges);
-    
+
     for (int i = 0; i < allConfs.length; i++) {
       setStageKeysFromBaseConf(allConfs[i], srcConf, Integer.toString(i));
       processDirectConversion(allConfs[i]);
     }
     for (int i = 0; i < allConfs.length - 1; i++) {
       processMultiStageDepreaction(allConfs[i], allConfs[i + 1]);
-      
+
     }
     // Unset unnecessary keys in the last stage. Will end up being called for
     // single stage as well which should be harmless.
@@ -163,29 +163,35 @@ public class MultiStageMRConfToTezTranslator {
     JobConf jobConf = new JobConf(baseConf);
     // Don't clobber explicit tez config.
     if (conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_INPUT_KEY_CLASS) == null
-        && conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_OUTPUT_KEY_CLASS) == null) {
+        && conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_OUTPUT_KEY_CLASS)
+        == null) {
       // If this is set, but the comparator is not set, and their types differ -
       // the job will break.
       if (conf.get(MRJobConfig.MAP_OUTPUT_KEY_CLASS) == null) {
         // Pull tis in from the baseConf
         conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, jobConf
             .getMapOutputKeyClass().getName());
-        LOG.info("XXX: Setting " + MRJobConfig.MAP_OUTPUT_KEY_CLASS
-            + " for stage: " + stage
-            + " based on job level configuration. Value: "
-            + conf.get(MRJobConfig.MAP_OUTPUT_KEY_CLASS));
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Setting " + MRJobConfig.MAP_OUTPUT_KEY_CLASS
+              + " for stage: " + stage
+              + " based on job level configuration. Value: "
+              + conf.get(MRJobConfig.MAP_OUTPUT_KEY_CLASS));
+        }
       }
     }
-    
+
     if (conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_INPUT_VALUE_CLASS) == null
-        && conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_OUTPUT_VALUE_CLASS) == null) {
+        && conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_OUTPUT_VALUE_CLASS)
+        == null) {
       if (conf.get(MRJobConfig.MAP_OUTPUT_VALUE_CLASS) == null) {
         conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, jobConf
             .getMapOutputValueClass().getName());
-        LOG.info("XXX: Setting " + MRJobConfig.MAP_OUTPUT_VALUE_CLASS
-            + " for stage: " + stage
-            + " based on job level configuration. Value: "
-            + conf.get(MRJobConfig.MAP_OUTPUT_VALUE_CLASS));
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Setting " + MRJobConfig.MAP_OUTPUT_VALUE_CLASS
+              + " for stage: " + stage
+              + " based on job level configuration. Value: "
+              + conf.get(MRJobConfig.MAP_OUTPUT_VALUE_CLASS));
+        }
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
index 276294c..5683fa1 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
@@ -76,16 +76,16 @@ import org.apache.tez.mapreduce.hadoop.mapred.TaskAttemptContextImpl;
 import org.apache.tez.mapreduce.hadoop.mapreduce.JobContextImpl;
 import org.apache.tez.mapreduce.partition.MRPartitioner;
 
-public abstract class MRTask 
+public abstract class MRTask
 extends RunningTaskContext {
 
   static final Log LOG = LogFactory.getLog(MRTask.class);
-  
+
   protected JobConf jobConf;
   protected JobContext jobContext;
   protected TaskAttemptContext taskAttemptContext;
   protected OutputCommitter committer;
-  
+
   // Current counters
   transient TezCounters counters = new TezCounters();
   protected GcTimeUpdater gcUpdater;
@@ -93,10 +93,10 @@ extends RunningTaskContext {
   private long initCpuCumulativeTime = 0;
   protected TezEngineTaskContext tezEngineTaskContext;
   protected TezTaskAttemptID taskAttemptId;
-  
+
   /* flag to track whether task is done */
   AtomicBoolean taskDone = new AtomicBoolean(false);
-  
+
   /** Construct output file names so that, when an output directory listing is
    * sorted lexicographically, positions correspond to output partitions.*/
   private static final NumberFormat NUMBER_FORMAT = NumberFormat.getInstance();
@@ -104,16 +104,16 @@ extends RunningTaskContext {
     NUMBER_FORMAT.setMinimumIntegerDigits(5);
     NUMBER_FORMAT.setGroupingUsed(false);
   }
-                          
+
   private final static int MAX_RETRIES = 10;
 
   /** The number of milliseconds between progress reports. */
   public static final int PROGRESS_INTERVAL = 3000;
 
   private MRTaskReporter mrReporter;
-  
+
   protected TaskSplitIndex splitMetaInfo = new TaskSplitIndex();
-  
+
   /**
    * A Map where Key-> URIScheme and value->FileSystemStatisticUpdater
    */
@@ -129,7 +129,7 @@ extends RunningTaskContext {
         new MRTaskStatus(
             taskAttemptId,
             counters,
-            (taskAttemptId.getTaskID().getVertexID().getId() == 0 ? 
+            (taskAttemptId.getTaskID().getVertexID().getId() == 0 ?
                 Phase.MAP : Phase.SHUFFLE)
         );
     gcUpdater = new GcTimeUpdater(counters);
@@ -143,15 +143,15 @@ extends RunningTaskContext {
     } else {
       this.jobConf = new JobConf(conf);
     }
-    reporter = 
+    reporter =
         new TezTaskReporterImpl(this, (TezTaskUmbilicalProtocol)master);
     ((TezTaskReporterImpl)reporter).startCommunicationThread();
-    
-    jobConf.set(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID, 
+
+    jobConf.set(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID,
         taskAttemptId.toString());
 
     initResourceCalculatorPlugin();
-    
+
     LOG.info("MRTask.inited: taskAttemptId = " + taskAttemptId.toString());
   }
 
@@ -171,7 +171,7 @@ extends RunningTaskContext {
   public TezTaskUmbilicalProtocol getUmbilical() {
     return ((TezTaskReporterImpl)reporter).getUmbilical();
   }
-  
+
   public void initTask(JobConf job, TezDAGID dagId,
       MRTaskReporter mrReporter,
       boolean useNewApi) throws IOException,
@@ -203,18 +203,18 @@ extends RunningTaskContext {
     } else {
       setCommitter(job.getOutputCommitter());
     }
-    
+
     Path outputPath = FileOutputFormat.getOutputPath(job);
     if (outputPath != null) {
       if ((getCommitter() instanceof FileOutputCommitter)) {
-        FileOutputFormat.setWorkOutputPath(job, 
+        FileOutputFormat.setWorkOutputPath(job,
             ((FileOutputCommitter)getCommitter()).getTaskAttemptPath(taskAttemptContext));
       } else {
         FileOutputFormat.setWorkOutputPath(job, outputPath);
       }
     }
     getCommitter().setupTask(taskAttemptContext);
-    
+
     partitioner = new MRPartitioner(this);
     ((MRPartitioner)partitioner).initialize(job, getTaskReporter());
 
@@ -244,7 +244,7 @@ extends RunningTaskContext {
 
   public void setState(State state) {
     // TODO Auto-generated method stub
-    
+
   }
 
   public State getState() {
@@ -261,22 +261,22 @@ extends RunningTaskContext {
   }
 
   public TezCounters getCounters() { return counters; }
-  
+
   /**
-   * Return current phase of the task. 
+   * Return current phase of the task.
    * needs to be synchronized as communication thread sends the phase every second
    * @return the curent phase of the task
    */
   public synchronized TezTaskStatus.Phase getPhase(){
-    return status.getPhase(); 
+    return status.getPhase();
   }
-  
+
   /**
-   * Set current phase of the task. 
-   * @param phase task phase 
+   * Set current phase of the task.
+   * @param phase task phase
    */
   protected synchronized void setPhase(TezTaskStatus.Phase phase){
-    status.setPhase(phase); 
+    status.setPhase(phase);
   }
 
   public void setConf(JobConf jobConf) {
@@ -290,9 +290,9 @@ extends RunningTaskContext {
   /**
    * Gets a handle to the Statistics instance based on the scheme associated
    * with path.
-   * 
+   *
    * @param path the path.
-   * @param conf the configuration to extract the scheme from if not part of 
+   * @param conf the configuration to extract the scheme from if not part of
    *   the path.
    * @return a Statistics instance, or null if none is found for the scheme.
    */
@@ -313,7 +313,7 @@ extends RunningTaskContext {
   public synchronized String getOutputName() {
     return "part-" + NUMBER_FORMAT.format(taskAttemptId.getTaskID().getId());
   }
- 
+
   public void waitBeforeCompletion(MRTaskReporter reporter) throws IOException,
       InterruptedException {
     TezTaskUmbilicalProtocol umbilical = getUmbilical();
@@ -400,7 +400,7 @@ extends RunningTaskContext {
         } catch (InterruptedException ie) {
           // ignore
         } catch (IOException ie) {
-          LOG.warn("Failure sending commit pending: " + 
+          LOG.warn("Failure sending commit pending: " +
               StringUtils.stringifyException(ie));
           if (--retries == 0) {
             System.exit(67);
@@ -421,7 +421,7 @@ extends RunningTaskContext {
     sendDone(umbilical);
   }
 
-  
+
   private boolean isCommitRequired() throws IOException {
     return committer.needsTaskCommit(taskAttemptContext);
   }
@@ -444,7 +444,7 @@ extends RunningTaskContext {
       } catch (InterruptedException ie) {
         Thread.currentThread().interrupt(); // interrupt ourself
       } catch (IOException ie) {
-        LOG.warn("Failure sending status update: " + 
+        LOG.warn("Failure sending status update: " +
             StringUtils.stringifyException(ie));
         if (--retries == 0) {
           throw ie;
@@ -454,9 +454,9 @@ extends RunningTaskContext {
   }
 
   /**
-   * Sends last status update before sending umbilical.done(); 
+   * Sends last status update before sending umbilical.done();
    */
-  private void sendLastUpdate() 
+  private void sendLastUpdate()
       throws IOException, InterruptedException {
     status.setOutputSize(-1l);
     // send a final status report
@@ -485,7 +485,7 @@ extends RunningTaskContext {
         }
         break;
       } catch (IOException ie) {
-        LOG.warn("Failure asking whether task can commit: " + 
+        LOG.warn("Failure asking whether task can commit: " +
             StringUtils.stringifyException(ie));
         if (--retries == 0) {
           //if it couldn't query successfully then delete the output
@@ -495,13 +495,13 @@ extends RunningTaskContext {
       }
     }
 
-    // task can Commit now  
+    // task can Commit now
     try {
       LOG.info("Task " + taskAttemptId + " is allowed to commit now");
       committer.commitTask(taskAttemptContext);
       return;
     } catch (IOException iee) {
-      LOG.warn("Failure committing: " + 
+      LOG.warn("Failure committing: " +
           StringUtils.stringifyException(iee));
       //if it couldn't commit a successfully then delete the output
       discardOutput(taskAttemptContext);
@@ -509,12 +509,12 @@ extends RunningTaskContext {
     }
   }
 
-  private 
+  private
   void discardOutput(TaskAttemptContext taskContext) {
     try {
       committer.abortTask(taskContext);
     } catch (IOException ioe)  {
-      LOG.warn("Failure cleaning up: " + 
+      LOG.warn("Failure cleaning up: " +
                StringUtils.stringifyException(ioe));
     }
   }
@@ -528,7 +528,7 @@ extends RunningTaskContext {
         LOG.info("Task '" + taskAttemptId + "' done.");
         return;
       } catch (IOException ie) {
-        LOG.warn("Failure signalling completion: " + 
+        LOG.warn("Failure signalling completion: " +
                  StringUtils.stringifyException(ie));
         if (--retries == 0) {
           throw ie;
@@ -540,7 +540,7 @@ extends RunningTaskContext {
   public void updateCounters() {
     // TODO Auto-generated method stub
     // TODO TEZAM Implement.
-    Map<String, List<FileSystem.Statistics>> map = new 
+    Map<String, List<FileSystem.Statistics>> map = new
         HashMap<String, List<FileSystem.Statistics>>();
     for(Statistics stat: FileSystem.getAllStatistics()) {
       String uriScheme = stat.getScheme();
@@ -563,7 +563,7 @@ extends RunningTaskContext {
       }
       updater.updateCounters();
     }
-    
+
     gcUpdater.incrementGcCounter();
     updateResourceCounters();
   }
@@ -599,7 +599,7 @@ extends RunningTaskContext {
     counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem);
     counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem);
   }
-  
+
 
   public static String normalizeStatus(String status, Configuration conf) {
     // Check to see if the status string is too long
@@ -614,75 +614,78 @@ extends RunningTaskContext {
     }
     return status;
   }
-  
-  protected static <INKEY,INVALUE,OUTKEY,OUTVALUE> 
+
+  protected static <INKEY,INVALUE,OUTKEY,OUTVALUE>
   org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE>.Context
   createReduceContext(org.apache.hadoop.mapreduce.Reducer
                         <INKEY,INVALUE,OUTKEY,OUTVALUE> reducer,
                       Configuration job,
-                      TezTaskAttemptID taskId, 
+                      TezTaskAttemptID taskId,
                       final TezRawKeyValueIterator rIter,
                       org.apache.hadoop.mapreduce.Counter inputKeyCounter,
                       org.apache.hadoop.mapreduce.Counter inputValueCounter,
-                      org.apache.hadoop.mapreduce.RecordWriter<OUTKEY,OUTVALUE> output, 
+                      org.apache.hadoop.mapreduce.RecordWriter<OUTKEY,OUTVALUE> output,
                       org.apache.hadoop.mapreduce.OutputCommitter committer,
                       org.apache.hadoop.mapreduce.StatusReporter reporter,
                       RawComparator<INKEY> comparator,
                       Class<INKEY> keyClass, Class<INVALUE> valueClass
   ) throws IOException, InterruptedException {
-    RawKeyValueIterator r = 
+    RawKeyValueIterator r =
         new RawKeyValueIterator() {
-          
+
           @Override
           public boolean next() throws IOException {
             return rIter.next();
           }
-          
+
           @Override
           public DataInputBuffer getValue() throws IOException {
             return rIter.getValue();
           }
-          
+
           @Override
           public Progress getProgress() {
             return rIter.getProgress();
           }
-          
+
           @Override
           public DataInputBuffer getKey() throws IOException {
             return rIter.getKey();
           }
-          
+
           @Override
           public void close() throws IOException {
             rIter.close();
           }
         };
-    org.apache.hadoop.mapreduce.ReduceContext<INKEY, INVALUE, OUTKEY, OUTVALUE> 
-    reduceContext = 
+    org.apache.hadoop.mapreduce.ReduceContext<INKEY, INVALUE, OUTKEY, OUTVALUE>
+    reduceContext =
       new ReduceContextImpl<INKEY, INVALUE, OUTKEY, OUTVALUE>(
-          job, 
-          IDConverter.toMRTaskAttemptId(taskId), 
-          r, 
-          inputKeyCounter, 
-          inputValueCounter, 
-          output, 
-          committer, 
-          reporter, 
-          comparator, 
-          keyClass, 
+          job,
+          IDConverter.toMRTaskAttemptId(taskId),
+          r,
+          inputKeyCounter,
+          inputValueCounter,
+          output,
+          committer,
+          reporter,
+          comparator,
+          keyClass,
           valueClass);
-    LOG.info("DEBUG: Using key class: " + keyClass + ", valueClass: " + valueClass);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Using key class: " + keyClass
+          + ", valueClass: " + valueClass);
+    }
 
-    org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE>.Context 
-        reducerContext = 
+    org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE>.Context
+        reducerContext =
           new WrappedReducer<INKEY, INVALUE, OUTKEY, OUTVALUE>().getReducerContext(
               reduceContext);
 
     return reducerContext;
   }
 
-  public void taskCleanup(TezTaskUmbilicalProtocol umbilical) 
+  public void taskCleanup(TezTaskUmbilicalProtocol umbilical)
       throws IOException, InterruptedException {
     // set phase for this task
     setPhase(TezTaskStatus.Phase.CLEANUP);
@@ -693,13 +696,13 @@ extends RunningTaskContext {
     committer.abortTask(taskAttemptContext);
   }
 
-  public void localizeConfiguration(JobConf jobConf) 
+  public void localizeConfiguration(JobConf jobConf)
       throws IOException, InterruptedException {
     jobConf.set(JobContext.TASK_ID, IDConverter
         .toMRTaskAttemptId(taskAttemptId).toString());
     jobConf.set(JobContext.TASK_ATTEMPT_ID,
         IDConverter.toMRTaskAttemptId(taskAttemptId).toString());
-    jobConf.setInt(JobContext.TASK_PARTITION, 
+    jobConf.setInt(JobContext.TASK_PARTITION,
         taskAttemptId.getTaskID().getId());
     jobConf.set(JobContext.ID, taskAttemptId.getTaskID().getVertexID().getDAGId().toString());
   }
@@ -727,11 +730,11 @@ extends RunningTaskContext {
   public JobContext getJobContext() {
     return jobContext;
   }
-  
+
   public TezTaskAttemptID getTaskAttemptId() {
     return taskAttemptId;
   }
-  
+
   public TezEngineTaskContext getTezEngineTaskContext() {
     return tezEngineTaskContext;
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index e2ef5b6..05b5411 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -322,12 +322,10 @@ public class YARNRunner implements ClientProtocol {
   private TaskLocationHint[] getMapLocationHintsFromInputSplits(JobID jobId,
       FileSystem fs, Configuration conf,
       String jobSubmitDir) throws IOException {
-    LOG.info("XXXX Reading splits information");
     TaskSplitMetaInfo[] splitsInfo =
         SplitMetaInfoReader.readSplitMetaInfo(jobId, fs, conf,
             new Path(jobSubmitDir));
     int splitsCount = splitsInfo.length;
-    LOG.info("XXXX Found splits information, splitCount=" + splitsCount);
     TaskLocationHint[] locationHints =
         new TaskLocationHint[splitsCount];
     for (int i = 0; i < splitsCount; ++i) {
@@ -462,11 +460,13 @@ public class YARNRunner implements ClientProtocol {
     Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
         getInitialClasspath(conf));
 
-    LOG.info("XXXX Dumping out env for child, isMap=" + isMap);
-    for (Map.Entry<String, String> entry : environment.entrySet()) {
-      LOG.info("XXXX env entry: "
-          + entry.getKey()
-          + "=" + entry.getValue());
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Dumping out env for child, isMap=" + isMap);
+      for (Map.Entry<String, String> entry : environment.entrySet()) {
+        LOG.debug("Child env entry: "
+            + entry.getKey()
+            + "=" + entry.getValue());
+      }
     }
   }
 
@@ -518,11 +518,13 @@ public class YARNRunner implements ClientProtocol {
           jobLocalResources, i);
       dag.addVertex(vertices[i]);
 
-      LOG.info("XXXX Adding intermediate vertex to DAG"
-          + ", vertexName=" + vertices[i].getVertexName()
-          + ", processor=" + vertices[i].getProcessorName()
-          + ", parrellism=" + vertices[i].getParallelism()
-          + ", javaOpts=" + vertices[i].getJavaOpts());
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Adding intermediate vertex to DAG"
+            + ", vertexName=" + vertices[i].getVertexName()
+            + ", processor=" + vertices[i].getProcessorName()
+            + ", parrellism=" + vertices[i].getParallelism()
+            + ", javaOpts=" + vertices[i].getJavaOpts());
+      }
     }
     return vertices;
   }
@@ -540,10 +542,12 @@ public class YARNRunner implements ClientProtocol {
 
     boolean isMRR = (intermediateReduces > 0);
 
-    LOG.info("XXXX Parsing job config"
-        + ", numMaps=" + numMaps
-        + ", numReduces=" + numReduces
-        + ", intermediateReduceStages=" + intermediateReduces);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Parsing job config"
+          + ", numMaps=" + numMaps
+          + ", numReduces=" + numReduces
+          + ", intermediateReduceStages=" + intermediateReduces);
+    }
 
     // configure map vertex
     String mapProcessor = "org.apache.tez.mapreduce.processor.map.MapProcessor";
@@ -575,11 +579,13 @@ public class YARNRunner implements ClientProtocol {
 
     mapVertex.setJavaOpts(getMapJavaOpts(jobConf));
 
-    LOG.info("XXXX Adding map vertex to DAG"
-        + ", vertexName=" + mapVertex.getVertexName()
-        + ", processor=" + mapVertex.getProcessorName()
-        + ", parrellism=" + mapVertex.getParallelism()
-        + ", javaOpts=" + mapVertex.getJavaOpts());
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Adding map vertex to DAG"
+          + ", vertexName=" + mapVertex.getVertexName()
+          + ", processor=" + mapVertex.getProcessorName()
+          + ", parrellism=" + mapVertex.getParallelism()
+          + ", javaOpts=" + mapVertex.getJavaOpts());
+    }
     dag.addVertex(mapVertex);
 
     Vertex[] intermediateVertices = null;
@@ -618,11 +624,13 @@ public class YARNRunner implements ClientProtocol {
 
       reduceVertex.setJavaOpts(getReduceJavaOpts(jobConf));
 
-      LOG.info("XXXX Adding reduce vertex to DAG"
-          + ", vertexName=" + reduceVertex.getVertexName()
-          + ", processor=" + reduceVertex.getProcessorName()
-          + ", parrellism=" + reduceVertex.getParallelism()
-          + ", javaOpts=" + reduceVertex.getJavaOpts());
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Adding reduce vertex to DAG"
+            + ", vertexName=" + reduceVertex.getVertexName()
+            + ", processor=" + reduceVertex.getProcessorName()
+            + ", parrellism=" + reduceVertex.getParallelism()
+            + ", javaOpts=" + reduceVertex.getJavaOpts());
+      }
       dag.addVertex(reduceVertex);
 
       EdgeProperty edgeProperty =
@@ -672,7 +680,9 @@ public class YARNRunner implements ClientProtocol {
     Map<String, String> mrParamToDAGParamMap = DeprecatedKeys.getMRToDAGParamMap();
     for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
       if (mrConf.get(entry.getKey()) != null) {
-        LOG.info("DEBUG: MR->DAG Setting new key: " + entry.getValue());
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("MR->DAG Setting new key: " + entry.getValue());
+        }
         dag.addConfiguration(entry.getValue(), mrConf.get(entry.getKey()));
       }
     }


[42/50] [abbrv] git commit: TEZ-26. Create tests for Vertex Scheduler (bikas)

Posted by ss...@apache.org.
TEZ-26. Create tests for Vertex Scheduler (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a3e2c2f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a3e2c2f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a3e2c2f5

Branch: refs/heads/master
Commit: a3e2c2f57af79dc6d5b84b07c699e3d96a64758c
Parents: d94d37a
Author: Bikas Saha <bi...@apache.org>
Authored: Sat Jun 1 00:00:06 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Sat Jun 1 00:00:06 2013 -0700

----------------------------------------------------------------------
 .../impl/BipartiteSlowStartVertexScheduler.java    |    8 +-
 .../tez/dag/app/dag/impl/TestDAGScheduler.java     |    4 +-
 .../tez/dag/app/dag/impl/TestVertexScheduler.java  |  181 +++++++++++++++
 3 files changed, 187 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a3e2c2f5/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
index 6c7bb0f..0321c10 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
@@ -49,7 +49,6 @@ public class BipartiteSlowStartVertexScheduler implements VertexScheduler {
   
   int numSourceTasks = 0;
   int numSourceTasksCompleted = 0;
-  boolean slowStartThresholdReached = false;
   ArrayList<TezTaskID> pendingTasks;
   int totalTasksToSchedule = 0;
   HashMap<TezVertexID, Vertex> bipartiteSources = 
@@ -84,7 +83,6 @@ public class BipartiteSlowStartVertexScheduler implements VertexScheduler {
   
   @Override
   public void onVertexStarted() {
-    //targetVertex.scheduleTasks(targetVertex.getTasks().keySet()); 
     pendingTasks = new ArrayList<TezTaskID>(managedVertex.getTotalTasks());
     // track the tasks in this vertex
     pendingTasks.addAll(managedVertex.getTasks().keySet());
@@ -140,7 +138,7 @@ public class BipartiteSlowStartVertexScheduler implements VertexScheduler {
     }
 
     float completedSourceTaskFraction = 0f;
-    if (numSourceTasks != 0) {//support for 0 source tasks
+    if (numSourceTasks != 0) { // support for 0 source tasks
       completedSourceTaskFraction = (float)numSourceTasksCompleted/numSourceTasks;
     } else {
       completedSourceTaskFraction = 1;
@@ -180,7 +178,9 @@ public class BipartiteSlowStartVertexScheduler implements VertexScheduler {
                managedVertex.getVertexId() + " with totalTasks: " + 
                totalTasksToSchedule + ". " + numSourceTasksCompleted + 
                " source tasks completed out of " + numSourceTasks + 
-               ". SourceTaskCompletedFraction: " + completedSourceTaskFraction);
+               ". SourceTaskCompletedFraction: " + completedSourceTaskFraction + 
+               " min: " + slowStartMinSrcCompletionFraction + 
+               " max: " + slowStartMaxSrcCompletionFraction);
       schedulePendingTasks(numTasksToSchedule);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a3e2c2f5/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
index 988a116..36c31a4 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
@@ -45,7 +45,7 @@ public class TestDAGScheduler {
   
   MockEventHandler mockEventHandler = new MockEventHandler();
   
-  //@Test(timeout=10000)
+  @Test(timeout=10000)
   public void testDAGSchedulerNaturalOrder() {
     DAG mockDag = mock(DAG.class);
     Vertex mockVertex = mock(Vertex.class);
@@ -72,7 +72,7 @@ public class TestDAGScheduler {
     Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 5);
   }
   
-  @Test//(timeout=10000)
+  @Test(timeout=10000)
   public void testDAGSchedulerMRR() {
     DAG mockDag = mock(DAG.class);
     TezDAGID dagId = new TezDAGID("1", 1, 1);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a3e2c2f5/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexScheduler.java
new file mode 100644
index 0000000..3ebb653
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexScheduler.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+
+import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.app.dag.Task;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import static org.mockito.Mockito.*;
+
+public class TestVertexScheduler {
+
+  @SuppressWarnings({ "unchecked", "rawtypes" })
+  @Test
+  public void testBipartiteSlowStartVertexScheduler() {
+    BipartiteSlowStartVertexScheduler scheduler = null;
+    TezDAGID dagId = new TezDAGID("1", 1, 1);
+    HashMap<Vertex, EdgeProperty> mockInputVertices = 
+        new HashMap<Vertex, EdgeProperty>();
+    Vertex mockSrcVertex1 = mock(Vertex.class);
+    TezVertexID mockSrcVertexId1 = new TezVertexID(dagId, 1);
+    EdgeProperty eProp1 = new EdgeProperty(
+        EdgeProperty.ConnectionPattern.BIPARTITE,
+        EdgeProperty.SourceType.STABLE, "in", "out");
+    when(mockSrcVertex1.getVertexId()).thenReturn(mockSrcVertexId1);
+    Vertex mockSrcVertex2 = mock(Vertex.class);
+    TezVertexID mockSrcVertexId2 = new TezVertexID(dagId, 2);
+    EdgeProperty eProp2 = new EdgeProperty(
+        EdgeProperty.ConnectionPattern.BIPARTITE,
+        EdgeProperty.SourceType.STABLE, "in", "out");
+    when(mockSrcVertex2.getVertexId()).thenReturn(mockSrcVertexId2);
+    Vertex mockSrcVertex3 = mock(Vertex.class);
+    TezVertexID mockSrcVertexId3 = new TezVertexID(dagId, 3);
+    EdgeProperty eProp3 = new EdgeProperty(
+        EdgeProperty.ConnectionPattern.ONE_TO_ALL,
+        EdgeProperty.SourceType.STABLE, "in", "out");
+    when(mockSrcVertex3.getVertexId()).thenReturn(mockSrcVertexId3);
+    
+    Vertex mockManagedVertex = mock(Vertex.class);
+    TezVertexID mockManagedVertexId = new TezVertexID(dagId, 3);
+    when(mockManagedVertex.getVertexId()).thenReturn(mockManagedVertexId);
+    when(mockManagedVertex.getInputVertices()).thenReturn(mockInputVertices);
+
+    // fail if there is no bipartite src vertex
+    mockInputVertices.put(mockSrcVertex3, eProp3);
+    try {
+     scheduler = new BipartiteSlowStartVertexScheduler(mockManagedVertex, 0, 0);
+     Assert.assertFalse(true);
+    } catch (TezException e) {
+      Assert.assertTrue(e.getMessage().contains(
+          "Atleast 1 bipartite source should exist"));
+    }
+    
+    mockInputVertices.put(mockSrcVertex1, eProp1);
+    mockInputVertices.put(mockSrcVertex2, eProp2);
+    
+    // check initialization
+    scheduler = 
+        new BipartiteSlowStartVertexScheduler(mockManagedVertex, 0.1f, 0.1f);
+    Assert.assertTrue(scheduler.bipartiteSources.size() == 2);
+    Assert.assertTrue(scheduler.bipartiteSources.containsKey(mockSrcVertexId1));
+    Assert.assertTrue(scheduler.bipartiteSources.containsKey(mockSrcVertexId2));
+    
+    HashMap<TezTaskID, Task> managedTasks = new HashMap<TezTaskID, Task>();
+    TezTaskID mockTaskId1 = new TezTaskID(mockManagedVertexId, 0);
+    managedTasks.put(mockTaskId1, null);
+    TezTaskID mockTaskId2 = new TezTaskID(mockManagedVertexId, 1);
+    managedTasks.put(mockTaskId2, null);
+    TezTaskID mockTaskId3 = new TezTaskID(mockManagedVertexId, 2);
+    managedTasks.put(mockTaskId3, null);
+    
+    when(mockManagedVertex.getTotalTasks()).thenReturn(3);
+    when(mockManagedVertex.getTasks()).thenReturn(managedTasks);
+    
+    final HashSet<TezTaskID> scheduledTasks = new HashSet<TezTaskID>();
+    doAnswer(new Answer() {
+      public Object answer(InvocationOnMock invocation) {
+          Object[] args = invocation.getArguments();
+          scheduledTasks.clear();
+          scheduledTasks.addAll((Collection<TezTaskID>)args[0]); 
+          return null;
+      }}).when(mockManagedVertex).scheduleTasks(anyCollection());
+    
+    // source vertices have 0 tasks. immediate start of all managed tasks
+    when(mockSrcVertex1.getTotalTasks()).thenReturn(0);
+    when(mockSrcVertex2.getTotalTasks()).thenReturn(0);
+    scheduler.onVertexStarted();
+    Assert.assertTrue(scheduler.pendingTasks.isEmpty());
+    Assert.assertTrue(scheduledTasks.size() == 3); // all tasks scheduled
+    
+    when(mockSrcVertex1.getTotalTasks()).thenReturn(2);
+    when(mockSrcVertex2.getTotalTasks()).thenReturn(2);
+    
+    // source vertex have some tasks. min, max == 0
+    scheduler = new BipartiteSlowStartVertexScheduler(mockManagedVertex, 0, 0);
+    scheduler.onVertexStarted();
+    Assert.assertTrue(scheduler.numSourceTasks == 4);
+    Assert.assertTrue(scheduler.totalTasksToSchedule == 3);
+    Assert.assertTrue(scheduler.numSourceTasksCompleted == 0);
+    Assert.assertTrue(scheduler.pendingTasks.isEmpty());
+    Assert.assertTrue(scheduledTasks.size() == 3); // all tasks scheduled
+    
+    TezTaskAttemptID mockSrcAttemptId11 = 
+        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId1, 0), 0);
+    TezTaskAttemptID mockSrcAttemptId12 = 
+        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId1, 0), 1);
+    TezTaskAttemptID mockSrcAttemptId21 = 
+        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId2, 0), 0);
+    TezTaskAttemptID mockSrcAttemptId22 = 
+        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId2, 0), 1);
+    TezTaskAttemptID mockSrcAttemptId31 = 
+        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId3, 0), 0);
+    
+    // min, max > 0 and min == max
+    scheduler = 
+        new BipartiteSlowStartVertexScheduler(mockManagedVertex, 0.25f, 0.25f);
+    scheduler.onVertexStarted();
+    Assert.assertTrue(scheduler.pendingTasks.size() == 3); // no tasks scheduled
+    Assert.assertTrue(scheduler.numSourceTasks == 4);
+    // task completion from non-bipartite stage does nothing
+    scheduler.onSourceTaskCompleted(mockSrcAttemptId31);
+    Assert.assertTrue(scheduler.pendingTasks.size() == 3); // no tasks scheduled
+    Assert.assertTrue(scheduler.numSourceTasks == 4);
+    Assert.assertTrue(scheduler.numSourceTasksCompleted == 0);
+    scheduler.onSourceTaskCompleted(mockSrcAttemptId11);
+    Assert.assertTrue(scheduler.pendingTasks.isEmpty());
+    Assert.assertTrue(scheduledTasks.size() == 3); // all tasks scheduled
+    Assert.assertTrue(scheduler.numSourceTasksCompleted == 1);
+    
+    // min, max > and min < max
+    scheduler = 
+        new BipartiteSlowStartVertexScheduler(mockManagedVertex, 0.25f, 0.75f);
+    scheduler.onVertexStarted();
+    Assert.assertTrue(scheduler.pendingTasks.size() == 3); // no tasks scheduled
+    Assert.assertTrue(scheduler.numSourceTasks == 4);
+    scheduler.onSourceTaskCompleted(mockSrcAttemptId11);
+    scheduler.onSourceTaskCompleted(mockSrcAttemptId12);
+    Assert.assertTrue(scheduler.pendingTasks.size() == 2);
+    Assert.assertTrue(scheduledTasks.size() == 1); // 1 task scheduled
+    Assert.assertTrue(scheduler.numSourceTasksCompleted == 2);
+    scheduler.onSourceTaskCompleted(mockSrcAttemptId21);
+    Assert.assertTrue(scheduler.pendingTasks.size() == 0);
+    Assert.assertTrue(scheduledTasks.size() == 2); // 2 tasks scheduled
+    Assert.assertTrue(scheduler.numSourceTasksCompleted == 3);
+    scheduledTasks.clear();
+    scheduler.onSourceTaskCompleted(mockSrcAttemptId22); // we are done. no action
+    Assert.assertTrue(scheduler.pendingTasks.size() == 0);
+    Assert.assertTrue(scheduledTasks.size() == 0); // no task scheduled
+    Assert.assertTrue(scheduler.numSourceTasksCompleted == 4);
+
+  }
+}


[43/50] [abbrv] git commit: TEZ-23. TaskScheduler should depend on official YARN AMRMClient* (bikas)

Posted by ss...@apache.org.
TEZ-23. TaskScheduler should depend on official YARN AMRMClient* (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/eeaaf232
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/eeaaf232
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/eeaaf232

Branch: refs/heads/master
Commit: eeaaf232f33a0b650ded7540c711d9d694e3157f
Parents: a3e2c2f
Author: Bikas Saha <bi...@apache.org>
Authored: Sat Jun 1 02:20:30 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Sat Jun 1 02:20:30 2013 -0700

----------------------------------------------------------------------
 .../java/org/apache/tez/dag/app/rm/AMRMClient.java |  182 ------
 .../org/apache/tez/dag/app/rm/AMRMClientAsync.java |  385 -------------
 .../org/apache/tez/dag/app/rm/AMRMClientImpl.java  |  447 ---------------
 .../org/apache/tez/dag/app/rm/TaskScheduler.java   |   91 ++--
 .../tez/dag/app/rm/TaskSchedulerEventHandler.java  |    5 +-
 .../apache/tez/dag/app/rm/TestTaskScheduler.java   |  103 +++-
 6 files changed, 131 insertions(+), 1082 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClient.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClient.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClient.java
deleted file mode 100644
index 7e098f1..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClient.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.rm;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.service.Service;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public interface AMRMClient<T extends AMRMClient.ContainerRequest> extends Service {
-
-  /**
-   * Object to represent container request for resources.
-   * Resources may be localized to nodes and racks.
-   * Resources may be assigned priorities.
-   * Can ask for multiple containers of a given type.
-   */
-  public static class ContainerRequest {
-    Resource capability;
-    String[] hosts;
-    String[] racks;
-    Priority priority;
-    int containerCount;
-        
-    public ContainerRequest(Resource capability, String[] hosts,
-        String[] racks, Priority priority, int containerCount) {
-      this.capability = capability;
-      this.hosts = (hosts != null ? hosts.clone() : null);
-      this.racks = (racks != null ? racks.clone() : null);
-      this.priority = priority;
-      this.containerCount = containerCount;
-    }
-    
-    public String toString() {
-      StringBuilder sb = new StringBuilder();
-      sb.append("Capability[").append(capability).append("]");
-      sb.append("Priority[").append(priority).append("]");
-      sb.append("ContainerCount[").append(containerCount).append("]");
-      return sb.toString();
-    }
-  }
-  
-  public static class StoredContainerRequest <T> extends ContainerRequest {
-    T cookie;
-    
-    public StoredContainerRequest(Resource capability, String[] hosts,
-        String[] racks, Priority priority) {
-      super(capability, hosts, racks, priority, 1);
-    }
-    
-    void setCookie(T cookie) {
-      this.cookie = cookie;
-    }
-    
-    T getCookie() {
-      return cookie;
-    }    
-  }
-  
-  /**
-   * Register the application master. This must be called before any 
-   * other interaction
-   * @param appHostName Name of the host on which master is running
-   * @param appHostPort Port master is listening on
-   * @param appTrackingUrl URL at which the master info can be seen
-   * @return <code>RegisterApplicationMasterResponse</code>
-   * @throws YarnRemoteException
-   * @throws IOException
-   */
-  public RegisterApplicationMasterResponse 
-               registerApplicationMaster(String appHostName,
-                                         int appHostPort,
-                                         String appTrackingUrl) 
-               throws YarnRemoteException, IOException;
-  
-  /**
-   * Request additional containers and receive new container allocations.
-   * Requests made via <code>addContainerRequest</code> are sent to the 
-   * <code>ResourceManager</code>. New containers assigned to the master are 
-   * retrieved. Status of completed containers and node health updates are 
-   * also retrieved.
-   * This also doubles up as a heartbeat to the ResourceManager and must be 
-   * made periodically.
-   * The call may not always return any new allocations of containers.
-   * App should not make concurrent allocate requests. May cause request loss.
-   * @param progressIndicator Indicates progress made by the master
-   * @return the response of the allocate request
-   * @throws YarnRemoteException
-   * @throws IOException
-   */
-  public AllocateResponse allocate(float progressIndicator) 
-                           throws YarnRemoteException, IOException;
-  
-  /**
-   * Unregister the application master. This must be called in the end.
-   * @param appStatus Success/Failure status of the master
-   * @param appMessage Diagnostics message on failure
-   * @param appTrackingUrl New URL to get master info
-   * @throws YarnRemoteException
-   * @throws IOException
-   */
-  public void unregisterApplicationMaster(FinalApplicationStatus appStatus,
-                                           String appMessage,
-                                           String appTrackingUrl) 
-               throws YarnRemoteException, IOException;
-  
-  /**
-   * Request containers for resources before calling <code>allocate</code>
-   * @param req Resource request
-   */
-  public void addContainerRequest(T req);
-  
-  /**
-   * Remove previous container request. The previous container request may have 
-   * already been sent to the ResourceManager. So even after the remove request 
-   * the app must be prepared to receive an allocation for the previous request 
-   * even after the remove request
-   * @param req Resource request
-   */
-  public void removeContainerRequest(T req);
-  
-  /**
-   * Release containers assigned by the Resource Manager. If the app cannot use
-   * the container or wants to give up the container then it can release them.
-   * The app needs to make new requests for the released resource capability if
-   * it still needs it. eg. it released non-local resources
-   * @param containerId
-   */
-  public void releaseAssignedContainer(ContainerId containerId);
-  
-  /**
-   * Get the currently available resources in the cluster.
-   * A valid value is available after a call to allocate has been made
-   * @return Currently available resources
-   */
-  public Resource getClusterAvailableResources();
-  
-  /**
-   * Get the current number of nodes in the cluster.
-   * A valid values is available after a call to allocate has been made
-   * @return Current number of nodes in the cluster
-   */
-  public int getClusterNodeCount();
-
-  /**
-   * Get outstanding <code>StoredContainerRequest</code>s matching the given 
-   * parameters. These StoredContainerRequests should have been added via
-   * <code>addContainerRequest</code> earlier in the lifecycle.
-   */
-  public Collection<T> getMatchingRequests(
-                                     Priority priority, 
-                                     String resourceName, 
-                                     Resource capability);
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientAsync.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientAsync.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientAsync.java
deleted file mode 100644
index b38183c..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientAsync.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.rm;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.service.AbstractService;
-import org.apache.tez.dag.app.rm.AMRMClient.ContainerRequest;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * <code>AMRMClientAsync</code> handles communication with the ResourceManager
- * and provides asynchronous updates on events such as container allocations and
- * completions.  It contains a thread that sends periodic heartbeats to the
- * ResourceManager.
- * 
- * It should be used by implementing a CallbackHandler:
- * <pre>
- * {@code
- * class MyCallbackHandler implements AMRMClientAsync.CallbackHandler {
- *   public void onContainersAllocated(List<Container> containers) {
- *     [run tasks on the containers]
- *   }
- *   
- *   public void onContainersCompleted(List<ContainerStatus> statuses) {
- *     [update progress, check whether app is done]
- *   }
- *   
- *   public void onNodesUpdated(List<NodeReport> updated) {}
- *   
- *   public void onReboot() {}
- * }
- * }
- * </pre>
- * 
- * The client's lifecycle should be managed similarly to the following:
- * 
- * <pre>
- * {@code
- * AMRMClientAsync asyncClient = new AMRMClientAsync(appAttId, 1000, new MyCallbackhandler());
- * asyncClient.init(conf);
- * asyncClient.start();
- * RegisterApplicationMasterResponse response = asyncClient
- *    .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
- *       appMasterTrackingUrl);
- * asyncClient.addContainerRequest(containerRequest);
- * [... wait for application to complete]
- * asyncClient.unregisterApplicationMaster(status, appMsg, trackingUrl);
- * asyncClient.stop();
- * }
- * </pre>
- */
-@Unstable
-@Evolving
-public class AMRMClientAsync<T extends ContainerRequest> extends AbstractService {
-  
-  private static final Log LOG = LogFactory.getLog(AMRMClientAsync.class);
-  
-  private final AMRMClient<T> client;
-  private final int intervalMs;
-  private final HeartbeatThread heartbeatThread;
-  private final CallbackHandlerThread handlerThread;
-  private final CallbackHandler handler;
-
-  private final BlockingQueue<AllocateResponse> responseQueue;
-  
-  private volatile boolean keepRunning;
-  private volatile float progress;
-  
-  private volatile Exception savedException;
-  
-  public AMRMClientAsync(ApplicationAttemptId id, int intervalMs,
-      CallbackHandler callbackHandler) {
-    this(new AMRMClientImpl<T>(id), intervalMs, callbackHandler);
-  }
-  
-  @Private
-  @VisibleForTesting
-  public AMRMClientAsync(AMRMClient<T> client, int intervalMs,
-      CallbackHandler callbackHandler) {
-    super(AMRMClientAsync.class.getName());
-    this.client = client;
-    this.intervalMs = intervalMs;
-    handler = callbackHandler;
-    heartbeatThread = new HeartbeatThread();
-    handlerThread = new CallbackHandlerThread();
-    responseQueue = new LinkedBlockingQueue<AllocateResponse>();
-    keepRunning = true;
-    savedException = null;
-  }
-    
-  @Override
-  public void init(Configuration conf) {
-    super.init(conf);
-    client.init(conf);
-  }  
-  
-  @Override
-  public void start() {
-    handlerThread.start();
-    client.start();
-    super.start();
-  }
-  
-  /**
-   * Tells the heartbeat and handler threads to stop and waits for them to
-   * terminate.  Calling this method from the callback handler thread would cause
-   * deadlock, and thus should be avoided.
-   */
-  @Override
-  public void stop() {
-    if (Thread.currentThread() == handlerThread) {
-      throw new YarnException("Cannot call stop from callback handler thread!");
-    }
-    keepRunning = false;
-    try {
-      heartbeatThread.join();
-    } catch (InterruptedException ex) {
-      LOG.error("Error joining with heartbeat thread", ex);
-    }
-    client.stop();
-    try {
-      handlerThread.interrupt();
-      handlerThread.join();
-    } catch (InterruptedException ex) {
-      LOG.error("Error joining with hander thread", ex);
-    }
-    super.stop();
-  }
-  
-  public Collection<T> getMatchingRequests(
-                                     Priority priority, 
-                                     String resourceName, 
-                                     Resource capability) {
-    return client.getMatchingRequests(priority, resourceName, capability);
-  }
-  
-  /**
-   * Registers this application master with the resource manager. On successful
-   * registration, starts the heartbeating thread.
-   * @throws YarnRemoteException
-   * @throws IOException
-   */
-  public RegisterApplicationMasterResponse registerApplicationMaster(
-      String appHostName, int appHostPort, String appTrackingUrl)
-      throws YarnRemoteException, IOException {
-    RegisterApplicationMasterResponse response =
-        client.registerApplicationMaster(appHostName, appHostPort, appTrackingUrl);
-    heartbeatThread.start();
-    return response;
-  }
-
-  /**
-   * Unregister the application master. This must be called in the end.
-   * @param appStatus Success/Failure status of the master
-   * @param appMessage Diagnostics message on failure
-   * @param appTrackingUrl New URL to get master info
-   * @throws YarnRemoteException
-   * @throws IOException
-   */
-  public void unregisterApplicationMaster(FinalApplicationStatus appStatus,
-      String appMessage, String appTrackingUrl) throws YarnRemoteException, IOException {
-    synchronized (client) {
-      keepRunning = false;
-      client.unregisterApplicationMaster(appStatus, appMessage, appTrackingUrl);
-    }
-  }
-
-  /**
-   * Request containers for resources before calling <code>allocate</code>
-   * @param req Resource request
-   */
-  public void addContainerRequest(T req) {
-    client.addContainerRequest(req);
-  }
-
-  /**
-   * Remove previous container request. The previous container request may have 
-   * already been sent to the ResourceManager. So even after the remove request 
-   * the app must be prepared to receive an allocation for the previous request 
-   * even after the remove request
-   * @param req Resource request
-   */
-  public void removeContainerRequest(T req) {
-    client.removeContainerRequest(req);
-  }
-
-  /**
-   * Release containers assigned by the Resource Manager. If the app cannot use
-   * the container or wants to give up the container then it can release them.
-   * The app needs to make new requests for the released resource capability if
-   * it still needs it. eg. it released non-local resources
-   * @param containerId
-   */
-  public void releaseAssignedContainer(ContainerId containerId) {
-    client.releaseAssignedContainer(containerId);
-  }
-
-  /**
-   * Get the currently available resources in the cluster.
-   * A valid value is available after a call to allocate has been made
-   * @return Currently available resources
-   */
-  public Resource getClusterAvailableResources() {
-    return client.getClusterAvailableResources();
-  }
-
-  /**
-   * Get the current number of nodes in the cluster.
-   * A valid values is available after a call to allocate has been made
-   * @return Current number of nodes in the cluster
-   */
-  public int getClusterNodeCount() {
-    return client.getClusterNodeCount();
-  }
-  
-  private class HeartbeatThread extends Thread {
-    public HeartbeatThread() {
-      super("AMRM Heartbeater thread");
-    }
-    
-    public void run() {
-      while (true) {
-        AllocateResponse response = null;
-        // synchronization ensures we don't send heartbeats after unregistering
-        synchronized (client) {
-          if (!keepRunning) {
-            break;
-          }
-            
-          try {
-            response = client.allocate(progress);
-          } catch (YarnRemoteException ex) {
-            LOG.error("Yarn exception on heartbeat", ex);
-            savedException = ex;
-            // interrupt handler thread in case it waiting on the queue
-            handlerThread.interrupt();
-            break;
-          } catch (IOException e) {
-            LOG.error("IO exception on heartbeat", e);
-            savedException = e;
-            // interrupt handler thread in case it waiting on the queue
-            handlerThread.interrupt();
-            break;
-          }
-        }
-        if (response != null) {
-          while (true) {
-            try {
-              responseQueue.put(response);
-              break;
-            } catch (InterruptedException ex) {
-              LOG.warn("Interrupted while waiting to put on response queue", ex);
-            }
-          }
-        }
-        
-        try {
-          Thread.sleep(intervalMs);
-        } catch (InterruptedException ex) {
-          LOG.warn("Heartbeater interrupted", ex);
-        }
-      }
-    }
-  }
-  
-  private class CallbackHandlerThread extends Thread {
-    public CallbackHandlerThread() {
-      super("AMRM Callback Handler Thread");
-    }
-    
-    public void run() {
-      while (keepRunning) {
-        AllocateResponse response;
-        try {
-          if(savedException != null) {
-            LOG.error("Stopping callback due to: ", savedException);
-            handler.onError(savedException);
-            break;
-          }
-          response = responseQueue.take();
-        } catch (InterruptedException ex) {
-          LOG.info("Interrupted while waiting for queue", ex);
-          continue;
-        }
-
-        if (response.getReboot()) {
-          handler.onRebootRequest();
-          LOG.info("Reboot requested. Stopping callback.");
-          break;
-        }
-        List<NodeReport> updatedNodes = response.getUpdatedNodes();
-        if (!updatedNodes.isEmpty()) {
-          handler.onNodesUpdated(updatedNodes);
-        }
-        
-        List<ContainerStatus> completed =
-            response.getCompletedContainersStatuses();
-        if (!completed.isEmpty()) {
-          handler.onContainersCompleted(completed);
-        }
-
-        List<Container> allocated = response.getAllocatedContainers();
-        if (!allocated.isEmpty()) {
-          handler.onContainersAllocated(allocated);
-        }
-        
-        progress = handler.getProgress();
-      }
-    }
-  }
-  
-  public interface CallbackHandler {
-    
-    /**
-     * Called when the ResourceManager responds to a heartbeat with completed
-     * containers. If the response contains both completed containers and
-     * allocated containers, this will be called before containersAllocated.
-     */
-    public void onContainersCompleted(List<ContainerStatus> statuses);
-    
-    /**
-     * Called when the ResourceManager responds to a heartbeat with allocated
-     * containers. If the response containers both completed containers and
-     * allocated containers, this will be called after containersCompleted.
-     */
-    public void onContainersAllocated(List<Container> containers);
-    
-    /**
-     * Called when the ResourceManager wants the ApplicationMaster to reboot
-     * for being out of sync. The ApplicationMaster should not unregister with 
-     * the RM unless the ApplicationMaster wants to be the last attempt.
-     */
-    public void onRebootRequest();
-    
-    /**
-     * Called when nodes tracked by the ResourceManager have changed in health,
-     * availability etc.
-     */
-    public void onNodesUpdated(List<NodeReport> updatedNodes);
-    
-    public float getProgress();
-    
-    public void onError(Exception e);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientImpl.java
deleted file mode 100644
index 75b9ad2..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientImpl.java
+++ /dev/null
@@ -1,447 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.rm;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedAction;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.api.AMRMProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.service.AbstractService;
-import org.apache.hadoop.yarn.util.BuilderUtils;
-
-// TODO check inputs for null etc. YARN-654
-
-@Unstable
-public class AMRMClientImpl<T extends org.apache.tez.dag.app.rm.AMRMClient.ContainerRequest> 
-                          extends AbstractService implements AMRMClient<T> {
-
-  private static final Log LOG = LogFactory.getLog(AMRMClientImpl.class);
-  
-  private final RecordFactory recordFactory =
-      RecordFactoryProvider.getRecordFactory(null);
-  
-  private int lastResponseId = 0;
-
-  protected AMRMProtocol rmClient;
-  protected final ApplicationAttemptId appAttemptId;  
-  protected Resource clusterAvailableResources;
-  protected int clusterNodeCount;
-  
-  class ResourceRequestInfo {
-    ResourceRequest remoteRequest;
-    HashSet<T> containerRequests;
-    
-    ResourceRequestInfo(Priority priority, String resourceName, Resource capability) {
-      remoteRequest = BuilderUtils.
-          newResourceRequest(priority, resourceName, capability, 0);
-      containerRequests = new HashSet<T>();
-    }
-  }
-  
-  //Key -> Priority
-  //Value -> Map
-  //Key->ResourceName (e.g., hostname, rackname, *)
-  //Value->Map
-  //Key->Resource Capability
-  //Value->ResourceRequest
-  protected final 
-  Map<Priority, Map<String, Map<Resource, ResourceRequestInfo>>>
-    remoteRequestsTable =
-    new TreeMap<Priority, Map<String, Map<Resource, ResourceRequestInfo>>>();
-
-  protected final Set<ResourceRequest> ask = new TreeSet<ResourceRequest>(
-      new org.apache.hadoop.yarn.util.BuilderUtils.ResourceRequestComparator());
-  protected final Set<ContainerId> release = new TreeSet<ContainerId>();
-  
-  public AMRMClientImpl(ApplicationAttemptId appAttemptId) {
-    super(AMRMClientImpl.class.getName());
-    this.appAttemptId = appAttemptId;
-  }
-
-  @Override
-  public synchronized void init(Configuration conf) {
-    super.init(conf);
-  }
-
-  @Override
-  public synchronized void start() {
-    final YarnConfiguration conf = new YarnConfiguration(getConfig());
-    final YarnRPC rpc = YarnRPC.create(conf);
-    final InetSocketAddress rmAddress = conf.getSocketAddr(
-        YarnConfiguration.RM_SCHEDULER_ADDRESS,
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
-
-    UserGroupInformation currentUser;
-    try {
-      currentUser = UserGroupInformation.getCurrentUser();
-    } catch (IOException e) {
-      throw new YarnException(e);
-    }
-
-    // CurrentUser should already have AMToken loaded.
-    rmClient = currentUser.doAs(new PrivilegedAction<AMRMProtocol>() {
-      @Override
-      public AMRMProtocol run() {
-        return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress,
-            conf);
-      }
-    });
-    LOG.debug("Connecting to ResourceManager at " + rmAddress);
-    super.start();
-  }
-
-  @Override
-  public synchronized void stop() {
-    if (this.rmClient != null) {
-      RPC.stopProxy(this.rmClient);
-    }
-    super.stop();
-  }
-  
-  @Override
-  public RegisterApplicationMasterResponse registerApplicationMaster(
-      String appHostName, int appHostPort, String appTrackingUrl)
-      throws YarnRemoteException, IOException {
-    // do this only once ???
-    RegisterApplicationMasterRequest request = recordFactory
-        .newRecordInstance(RegisterApplicationMasterRequest.class);
-    synchronized (this) {
-      request.setApplicationAttemptId(appAttemptId);      
-    }
-    request.setHost(appHostName);
-    request.setRpcPort(appHostPort);
-    if(appTrackingUrl != null) {
-      request.setTrackingUrl(appTrackingUrl);
-    }
-    RegisterApplicationMasterResponse response = rmClient
-        .registerApplicationMaster(request);
-    return response;
-  }
-
-  @Override
-  public AllocateResponse allocate(float progressIndicator) 
-      throws YarnRemoteException, IOException {
-    AllocateResponse allocateResponse = null;
-    ArrayList<ResourceRequest> askList = null;
-    ArrayList<ContainerId> releaseList = null;
-    AllocateRequest allocateRequest = null;
-    
-    try {
-      synchronized (this) {
-        askList = new ArrayList<ResourceRequest>(ask);
-        releaseList = new ArrayList<ContainerId>(release);
-        // optimistically clear this collection assuming no RPC failure
-        ask.clear();
-        release.clear();
-        allocateRequest = BuilderUtils
-            .newAllocateRequest(appAttemptId, lastResponseId, progressIndicator,
-                askList, releaseList);
-      }
-
-      allocateResponse = rmClient.allocate(allocateRequest);
-
-      synchronized (this) {
-        // update these on successful RPC
-        clusterNodeCount = allocateResponse.getNumClusterNodes();
-        lastResponseId = allocateResponse.getResponseId();
-        clusterAvailableResources = allocateResponse.getAvailableResources();
-      }
-    } finally {
-      // TODO how to differentiate remote yarn exception vs error in rpc
-      if(allocateResponse == null) {
-        // we hit an exception in allocate()
-        // preserve ask and release for next call to allocate()
-        synchronized (this) {
-          release.addAll(releaseList);
-          // requests could have been added or deleted during call to allocate
-          // If requests were added/removed then there is nothing to do since
-          // the ResourceRequest object in ask would have the actual new value.
-          // If ask does not have this ResourceRequest then it was unchanged and
-          // so we can add the value back safely.
-          // This assumes that there will no concurrent calls to allocate() and
-          // so we dont have to worry about ask being changed in the
-          // synchronized block at the beginning of this method.
-          for(ResourceRequest oldAsk : askList) {
-            if(!ask.contains(oldAsk)) {
-              ask.add(oldAsk);
-            }
-          }
-        }
-      }
-    }
-    return allocateResponse;
-  }
-
-  @Override
-  public void unregisterApplicationMaster(FinalApplicationStatus appStatus,
-      String appMessage, String appTrackingUrl) throws YarnRemoteException,
-      IOException {
-    FinishApplicationMasterRequest request = recordFactory
-                  .newRecordInstance(FinishApplicationMasterRequest.class);
-    request.setAppAttemptId(appAttemptId);
-    request.setFinishApplicationStatus(appStatus);
-    if(appMessage != null) {
-      request.setDiagnostics(appMessage);
-    }
-    if(appTrackingUrl != null) {
-      request.setTrackingUrl(appTrackingUrl);
-    }
-    rmClient.finishApplicationMaster(request);
-  }
-  
-  @Override
-  public synchronized void addContainerRequest(T req) {
-    // Create resource requests
-    // add check for dup locations
-    if(req.hosts != null) {
-      for (String host : req.hosts) {
-        addResourceRequest(req.priority, host, req.capability, req.containerCount, req);
-      }
-    }
-
-    if(req.racks != null) {
-      for (String rack : req.racks) {
-        addResourceRequest(req.priority, rack, req.capability, req.containerCount, req);
-      }
-    }
-
-    // Off-switch
-    addResourceRequest(req.priority, ResourceRequest.ANY, req.capability, req.containerCount, req); 
-  }
-
-  @Override
-  public synchronized void removeContainerRequest(T req) {
-    // Update resource requests
-    if(req.hosts != null) {
-      for (String hostName : req.hosts) {
-        decResourceRequest(req.priority, hostName, req.capability, req.containerCount, req);
-      }
-    }
-    
-    if(req.racks != null) {
-      for (String rack : req.racks) {
-        decResourceRequest(req.priority, rack, req.capability, req.containerCount, req);
-      }
-    }
-   
-    decResourceRequest(req.priority, ResourceRequest.ANY, req.capability, req.containerCount, req);
-    
-  }
-
-  @Override
-  public synchronized void releaseAssignedContainer(ContainerId containerId) {
-    release.add(containerId);
-  }
-  
-  @Override
-  public synchronized Resource getClusterAvailableResources() {
-    return clusterAvailableResources;
-  }
-  
-  @Override
-  public synchronized int getClusterNodeCount() {
-    return clusterNodeCount;
-  }
-  
-  @Override
-  public synchronized Collection<T> getMatchingRequests(
-                                          Priority priority, 
-                                          String resourceName, 
-                                          Resource capability) {
-    Map<String, Map<Resource, ResourceRequestInfo>> remoteRequests = 
-        this.remoteRequestsTable.get(priority);
-    if (remoteRequests == null) {
-      return null;
-    }
-    Map<Resource, ResourceRequestInfo> reqMap = remoteRequests
-        .get(resourceName);
-    if (reqMap == null) {
-      return null;
-    }
-    ResourceRequestInfo resourceRequestInfo = reqMap.get(capability);
-    if (resourceRequestInfo == null) {
-      return null;
-    }
-    
-    return resourceRequestInfo.containerRequests;
-  }
-  
-  private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
-    // This code looks weird but is needed because of the following scenario.
-    // A ResourceRequest is removed from the remoteRequestTable. A 0 container 
-    // request is added to 'ask' to notify the RM about not needing it any more.
-    // Before the call to allocate, the user now requests more containers. If 
-    // the locations of the 0 size request and the new request are the same
-    // (with the difference being only container count), then the set comparator
-    // will consider both to be the same and not add the new request to ask. So 
-    // we need to check for the "same" request being present and remove it and 
-    // then add it back. The comparator is container count agnostic.
-    // This should happen only rarely but we do need to guard against it.
-    if(ask.contains(remoteRequest)) {
-      ask.remove(remoteRequest);
-    }
-    ask.add(remoteRequest);
-  }
-
-  private void addResourceRequest(Priority priority, String resourceName,
-      Resource capability, int containerCount, T req) {
-    Map<String, Map<Resource, ResourceRequestInfo>> remoteRequests =
-      this.remoteRequestsTable.get(priority);
-    if (remoteRequests == null) {
-      remoteRequests = new HashMap<String, Map<Resource, ResourceRequestInfo>>();
-      this.remoteRequestsTable.put(priority, remoteRequests);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Added priority=" + priority);
-      }
-    }
-    Map<Resource, ResourceRequestInfo> reqMap = remoteRequests.get(resourceName);
-    if (reqMap == null) {
-      reqMap = new HashMap<Resource, ResourceRequestInfo>();
-      remoteRequests.put(resourceName, reqMap);
-    }
-    ResourceRequestInfo resourceRequestInfo = reqMap.get(capability);
-    if (resourceRequestInfo == null) {
-      resourceRequestInfo =
-          new ResourceRequestInfo(priority, resourceName, capability);
-      reqMap.put(capability, resourceRequestInfo);
-    }
-    
-    resourceRequestInfo.remoteRequest.setNumContainers(
-         resourceRequestInfo.remoteRequest.getNumContainers() + containerCount);
-
-    if(req instanceof StoredContainerRequest<?>) {
-      resourceRequestInfo.containerRequests.add(req);
-    }
-
-    // Note this down for next interaction with ResourceManager
-    addResourceRequestToAsk(resourceRequestInfo.remoteRequest);
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("addResourceRequest:" + " applicationId="
-          + appAttemptId + " priority=" + priority.getPriority()
-          + " resourceName=" + resourceName + " numContainers="
-          + resourceRequestInfo.remoteRequest.getNumContainers() 
-          + " #asks=" + ask.size());
-    }
-  }
-
-  private void decResourceRequest(Priority priority, 
-                                   String resourceName,
-                                   Resource capability, 
-                                   int containerCount, 
-                                   T req) {
-    Map<String, Map<Resource, ResourceRequestInfo>> remoteRequests =
-      this.remoteRequestsTable.get(priority);
-    
-    if(remoteRequests == null) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Not decrementing resource as priority " + priority 
-            + " is not present in request table");
-      }
-      return;
-    }
-    
-    Map<Resource, ResourceRequestInfo> reqMap = remoteRequests.get(resourceName);
-    if (reqMap == null) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Not decrementing resource as " + resourceName
-            + " is not present in request table");
-      }
-      return;
-    }
-    ResourceRequestInfo resourceRequestInfo = reqMap.get(capability);
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("BEFORE decResourceRequest:" + " applicationId="
-          + appAttemptId + " priority=" + priority.getPriority()
-          + " resourceName=" + resourceName + " numContainers="
-          + resourceRequestInfo.remoteRequest.getNumContainers() 
-          + " #asks=" + ask.size());
-    }
-
-    resourceRequestInfo.remoteRequest.setNumContainers(
-        resourceRequestInfo.remoteRequest.getNumContainers() - containerCount);
-
-    if(req instanceof StoredContainerRequest<?>) {
-      resourceRequestInfo.containerRequests.remove(req);
-    }
-    
-    if(resourceRequestInfo.remoteRequest.getNumContainers() < 0) {
-      // guard against spurious removals
-      resourceRequestInfo.remoteRequest.setNumContainers(0);
-    }
-    // send the ResourceRequest to RM even if is 0 because it needs to override
-    // a previously sent value. If ResourceRequest was not sent previously then
-    // sending 0 aught to be a no-op on RM
-    addResourceRequestToAsk(resourceRequestInfo.remoteRequest);
-
-    // delete entries from map if no longer needed
-    if (resourceRequestInfo.remoteRequest.getNumContainers() == 0) {
-      reqMap.remove(capability);
-      if (reqMap.size() == 0) {
-        remoteRequests.remove(resourceName);
-      }
-      if (remoteRequests.size() == 0) {
-        remoteRequestsTable.remove(priority);
-      }
-    }
-
-    if (LOG.isDebugEnabled()) {
-      LOG.info("AFTER decResourceRequest:" + " applicationId="
-          + appAttemptId + " priority=" + priority.getPriority()
-          + " resourceName=" + resourceName + " numContainers="
-          + resourceRequestInfo.remoteRequest.getNumContainers() 
-          + " #asks=" + ask.size());
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
index 14188b0..d081d2a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
@@ -42,11 +42,12 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.client.AMRMClient.StoredContainerRequest;
+import org.apache.hadoop.yarn.client.AMRMClientAsync;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.hadoop.yarn.util.RackResolver;
 import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.app.rm.AMRMClient.StoredContainerRequest;
 import org.apache.tez.dag.app.rm.TaskScheduler.TaskSchedulerAppCallback.AppFinalStatus;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -98,11 +99,11 @@ public class TaskScheduler extends AbstractService
     public AppFinalStatus getFinalAppStatus();
   }
   
-  final AMRMClientAsync<StoredContainerRequest<CRCookie>> amRmClient;
+  final AMRMClientAsync<CookieContainerRequest> amRmClient;
   final TaskSchedulerAppCallback appClient;
   
-  Map<Object, StoredContainerRequest<CRCookie>> taskRequests =  
-                  new HashMap<Object, StoredContainerRequest<CRCookie>>();
+  Map<Object, CookieContainerRequest> taskRequests =  
+                  new HashMap<Object, CookieContainerRequest>();
   Map<Object, Container> taskAllocations = 
                   new HashMap<Object, Container>();
   Map<ContainerId, Object> containerAssigments = 
@@ -121,6 +122,19 @@ public class TaskScheduler extends AbstractService
     Object appCookie;
   }
   
+  class CookieContainerRequest extends StoredContainerRequest {
+    CRCookie cookie;
+    public CookieContainerRequest(Resource capability, String[] hosts,
+        String[] racks, Priority priority, CRCookie cookie) {
+      super(capability, hosts, racks, priority);
+      this.cookie = cookie;
+    }
+    
+    CRCookie getCookie() {
+      return cookie;
+    }
+  }
+  
   public TaskScheduler(ApplicationAttemptId id, 
                         TaskSchedulerAppCallback appClient,
                         String appHostName, 
@@ -129,7 +143,7 @@ public class TaskScheduler extends AbstractService
     super(TaskScheduler.class.getName());
     this.appClient = appClient;
     this.amRmClient = 
-        new AMRMClientAsync<StoredContainerRequest<CRCookie>>(id, 1000, this);
+        new AMRMClientAsync<CookieContainerRequest>(id, 1000, this);
     this.appHostName = appHostName;
     this.appHostPort = appHostPort;
     this.appTrackingUrl = appTrackingUrl;
@@ -142,7 +156,7 @@ public class TaskScheduler extends AbstractService
       String appHostName, 
       int appHostPort,
       String appTrackingUrl,
-      AMRMClientAsync<StoredContainerRequest<CRCookie>> client) {
+      AMRMClientAsync<CookieContainerRequest> client) {
     super(TaskScheduler.class.getName());
     this.appClient = appClient;
     this.amRmClient = client;
@@ -270,13 +284,12 @@ public class TaskScheduler extends AbstractService
     if(isStopped) {
       return;
     }
-    Map<StoredContainerRequest<CRCookie>, Container> appContainers = 
-        new HashMap<StoredContainerRequest<CRCookie>, Container>
-                                                        (containers.size());
+    Map<CookieContainerRequest, Container> appContainers = 
+        new HashMap<CookieContainerRequest, Container>(containers.size());
     synchronized (this) {
       for(Container container : containers) {
         String location = container.getNodeId().getHost();
-        StoredContainerRequest<CRCookie> assigned = getMatchingRequest(container, location);
+        CookieContainerRequest assigned = getMatchingRequest(container, location);
         if(assigned == null) {
           location = RackResolver.resolve(location).getNetworkLocation();
           assigned = getMatchingRequest(container, location);
@@ -307,10 +320,10 @@ public class TaskScheduler extends AbstractService
     }
     
     // upcall to app must be outside locks
-    for (Entry<StoredContainerRequest<CRCookie>, Container> entry : 
-              appContainers.entrySet()) {
-      StoredContainerRequest<CRCookie> assigned = entry.getKey();
-      appClient.taskAllocated(getTask(assigned), getAppCookie(assigned),
+    for (Entry<CookieContainerRequest, Container> entry : 
+                                        appContainers.entrySet()) {
+      CookieContainerRequest assigned = entry.getKey();
+      appClient.taskAllocated(getTask(assigned), assigned.getCookie().appCookie,
           entry.getValue());
     }   
   }
@@ -357,16 +370,16 @@ public class TaskScheduler extends AbstractService
                                            Priority priority,
                                            Object clientCookie) {
     // TODO check for nulls etc
-    StoredContainerRequest<CRCookie> request = 
-             new StoredContainerRequest<CRCookie>(capability, 
-                                                   hosts, 
-                                                   racks, 
-                                                   priority);
     // TODO extra memory allocation
     CRCookie cookie = new CRCookie();
     cookie.task = task;
     cookie.appCookie = clientCookie;
-    request.setCookie(cookie);
+    CookieContainerRequest request = 
+             new CookieContainerRequest(capability, 
+                                         hosts, 
+                                         racks, 
+                                         priority,
+                                         cookie);
 
     addTaskRequest(task, request);
     LOG.info("Allocation request for task: " + task + 
@@ -374,7 +387,7 @@ public class TaskScheduler extends AbstractService
   }
   
   public synchronized Container deallocateTask(Object task) {
-    StoredContainerRequest<CRCookie> request = removeTaskRequest(task);
+    CookieContainerRequest request = removeTaskRequest(task);
     if(request != null) {
       // task not allocated yet
       LOG.info("Deallocating task: " + task + " before allocation");
@@ -406,31 +419,29 @@ public class TaskScheduler extends AbstractService
     return null;
   }
   
-  private StoredContainerRequest<CRCookie> getMatchingRequest(
+  private CookieContainerRequest getMatchingRequest(
                                       Container container, String location) {
     Priority priority = container.getPriority();
     Resource capability = container.getResource();
-    StoredContainerRequest<CRCookie> assigned = null;
-    Collection<StoredContainerRequest<CRCookie>> requests =
+    CookieContainerRequest assigned = null;
+    List<? extends Collection<CookieContainerRequest>> requestsList =
         amRmClient.getMatchingRequests(priority, location, capability);
     
-    if(requests != null) {
-      // TODO maybe do FIFO
-      Iterator<StoredContainerRequest<CRCookie>> iterator = requests.iterator();
-      if(iterator.hasNext()) {
-        assigned = requests.iterator().next();
+    if(requestsList.size() > 0) {
+      // pick first one
+      for(Collection<CookieContainerRequest> requests : requestsList) {
+        Iterator<CookieContainerRequest> iterator = requests.iterator();
+        if(iterator.hasNext()) {
+          assigned = requests.iterator().next();
+        }
       }
     }
     
     return assigned;
   }
   
-  private Object getTask(StoredContainerRequest<CRCookie> request) {
-    return ((CRCookie)request.getCookie()).task;
-  }
-  
-  private Object getAppCookie(StoredContainerRequest<CRCookie> request) {
-    return ((CRCookie)request.getCookie()).appCookie;
+  private Object getTask(CookieContainerRequest request) {
+    return request.getCookie().task;
   }
   
   private void releaseContainer(ContainerId containerId, Object task) {
@@ -442,8 +453,8 @@ public class TaskScheduler extends AbstractService
   
   private void assignContainer(Object task, 
                                 Container container, 
-                                StoredContainerRequest<CRCookie> assigned) {
-    StoredContainerRequest<CRCookie> request = removeTaskRequest(task);
+                                CookieContainerRequest assigned) {
+    CookieContainerRequest request = removeTaskRequest(task);
     assert request != null;
     //assert assigned.equals(request);
 
@@ -453,8 +464,8 @@ public class TaskScheduler extends AbstractService
     
   }
   
-  private StoredContainerRequest<CRCookie> removeTaskRequest(Object task) {
-    StoredContainerRequest<CRCookie> request = taskRequests.remove(task);
+  private CookieContainerRequest removeTaskRequest(Object task) {
+    CookieContainerRequest request = taskRequests.remove(task);
     if(request != null) {
       // remove all references of the request from AMRMClient
       amRmClient.removeContainerRequest(request);
@@ -463,7 +474,7 @@ public class TaskScheduler extends AbstractService
   }
   
   private void addTaskRequest(Object task, 
-                                StoredContainerRequest<CRCookie> request) {
+                                CookieContainerRequest request) {
     // TODO TEZ-37 fix duplicate handling
     taskRequests.put(task, request);
     amRmClient.addContainerRequest(request);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 8b98c68..e29db1c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -496,8 +496,6 @@ public class TaskSchedulerEventHandler extends AbstractService
     // TODO TEZ-34 change event to reboot and send to app master
     sendEvent(new DAGEvent(appContext.getDAGID(),
                            DAGEventType.INTERNAL_ERROR));
-    throw new YarnException("ResourceManager requests reboot for: "
-                             + appContext.getApplicationID());  
   }
 
   @Override
@@ -552,7 +550,8 @@ public class TaskSchedulerEventHandler extends AbstractService
 
   @Override
   public void onError(Exception e) {
-    // TODO TEZ-35 handle error
+    sendEvent(new DAGEvent(appContext.getDAGID(),
+        DAGEventType.INTERNAL_ERROR));
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
index 4fecd3a..8d52e9a 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
@@ -19,6 +19,8 @@
 package org.apache.tez.dag.app.rm;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
@@ -34,18 +36,20 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.client.AMRMClientAsync;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.RackResolver;
-import org.apache.tez.dag.app.rm.AMRMClient.StoredContainerRequest;
-import org.apache.tez.dag.app.rm.TaskScheduler.CRCookie;
+import org.apache.tez.dag.app.rm.TaskScheduler.CookieContainerRequest;
 import org.apache.tez.dag.app.rm.TaskScheduler.TaskSchedulerAppCallback;
 import org.apache.tez.dag.app.rm.TaskScheduler.TaskSchedulerAppCallback.AppFinalStatus;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 
 import static org.mockito.Mockito.*;
 
@@ -54,13 +58,13 @@ public class TestTaskScheduler {
   RecordFactory recordFactory =
       RecordFactoryProvider.getRecordFactory(null);
     
-  @SuppressWarnings({ "unchecked", "rawtypes" })
+  @SuppressWarnings({ "unchecked" })
   @Test
   public void testTaskScheduler() throws Exception {
     RackResolver.init(new YarnConfiguration());
     TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class);
     
-    AMRMClientAsync<StoredContainerRequest<CRCookie>> mockRMClient = 
+    AMRMClientAsync<CookieContainerRequest> mockRMClient = 
                                                   mock(AMRMClientAsync.class);
     
     ApplicationAttemptId attemptId = 
@@ -111,25 +115,25 @@ public class TestTaskScheduler {
     String[] hosts = {"host1", "host5"};
     String[] racks = {"/default-rack", "/default-rack"};
     Priority mockPriority = mock(Priority.class);
-    ArgumentCaptor<StoredContainerRequest> requestCaptor = 
-                        ArgumentCaptor.forClass(StoredContainerRequest.class);
+    ArgumentCaptor<CookieContainerRequest> requestCaptor = 
+                        ArgumentCaptor.forClass(CookieContainerRequest.class);
     // allocate task
     scheduler.allocateTask(mockTask1, mockCapability, hosts, 
                            racks, mockPriority, mockCookie1);
     verify(mockRMClient, times(1)).
-                           addContainerRequest((StoredContainerRequest) any());
+                           addContainerRequest((CookieContainerRequest) any());
 
     // returned from task requests before allocation happens
     Assert.assertNull(scheduler.deallocateTask(mockTask1));
     verify(mockRMClient, times(1)).
-                        removeContainerRequest((StoredContainerRequest) any());
+                        removeContainerRequest((CookieContainerRequest) any());
     verify(mockRMClient, times(0)).
                                  releaseAssignedContainer((ContainerId) any());
     
     // deallocating unknown task
     Assert.assertNull(scheduler.deallocateTask(mockTask1));
     verify(mockRMClient, times(1)).
-                        removeContainerRequest((StoredContainerRequest) any());
+                        removeContainerRequest((CookieContainerRequest) any());
     verify(mockRMClient, times(0)).
                                  releaseAssignedContainer((ContainerId) any());
 
@@ -142,17 +146,17 @@ public class TestTaskScheduler {
         racks, mockPriority, mockCookie1);
     verify(mockRMClient, times(2)).
                                 addContainerRequest(requestCaptor.capture());
-    StoredContainerRequest<CRCookie> request1 = requestCaptor.getValue();
+    CookieContainerRequest request1 = requestCaptor.getValue();
     scheduler.allocateTask(mockTask2, mockCapability, hosts, 
         racks, mockPriority, mockCookie2);
     verify(mockRMClient, times(3)).
                                 addContainerRequest(requestCaptor.capture());
-    StoredContainerRequest<CRCookie> request2 = requestCaptor.getValue();
+    CookieContainerRequest request2 = requestCaptor.getValue();
     scheduler.allocateTask(mockTask3, mockCapability, hosts, 
         racks, mockPriority, mockCookie3);
     verify(mockRMClient, times(4)).
                                 addContainerRequest(requestCaptor.capture());
-    StoredContainerRequest<CRCookie> request3 = requestCaptor.getValue();
+    CookieContainerRequest request3 = requestCaptor.getValue();
     
     List<Container> containers = new ArrayList<Container>();
     Container mockContainer1 = mock(Container.class, RETURNS_DEEP_STUBS);
@@ -175,35 +179,84 @@ public class TestTaskScheduler {
     ContainerId mockCId4 = mock(ContainerId.class);
     when(mockContainer4.getId()).thenReturn(mockCId4);
     containers.add(mockContainer4);
-    ArrayList<StoredContainerRequest<CRCookie>> hostContainers = 
-                             new ArrayList<StoredContainerRequest<CRCookie>>();
+    ArrayList<CookieContainerRequest> hostContainers = 
+                             new ArrayList<CookieContainerRequest>();
     hostContainers.add(request1);
     hostContainers.add(request2);
     hostContainers.add(request3);
-    ArrayList<StoredContainerRequest<CRCookie>> rackContainers = 
-                             new ArrayList<StoredContainerRequest<CRCookie>>();
+    ArrayList<CookieContainerRequest> rackContainers = 
+                             new ArrayList<CookieContainerRequest>();
     rackContainers.add(request2);
     rackContainers.add(request3);
-    ArrayList<StoredContainerRequest<CRCookie>> anyContainers = 
-                             new ArrayList<StoredContainerRequest<CRCookie>>();
+    ArrayList<CookieContainerRequest> anyContainers = 
+                             new ArrayList<CookieContainerRequest>();
     anyContainers.add(request3);
 
+    final List<ArrayList<CookieContainerRequest>> hostList = 
+                        new LinkedList<ArrayList<CookieContainerRequest>>();
+    hostList.add(hostContainers);
+    final List<ArrayList<CookieContainerRequest>> rackList = 
+                        new LinkedList<ArrayList<CookieContainerRequest>>();
+    rackList.add(rackContainers);
+    final List<ArrayList<CookieContainerRequest>> anyList = 
+                        new LinkedList<ArrayList<CookieContainerRequest>>();
+    anyList.add(anyContainers);
+    final List<ArrayList<CookieContainerRequest>> emptyList = 
+                        new LinkedList<ArrayList<CookieContainerRequest>>();
     // return all requests for host1
     when(
         mockRMClient.getMatchingRequests((Priority) any(), eq("host1"),
-            (Resource) any())).thenReturn(hostContainers);
+            (Resource) any())).thenAnswer(
+        new Answer<List<? extends Collection<CookieContainerRequest>>>() {
+          @Override
+          public List<? extends Collection<CookieContainerRequest>> answer(
+              InvocationOnMock invocation) throws Throwable {
+            return hostList;
+          }
+
+        });
     // first request matched by host
-    // second request matched to rack. RackResolver by default puts hosts in 
+    // second request matched to rack. RackResolver by default puts hosts in
     // /default-rack. We need to workaround by returning rack matches only once
     when(
-        mockRMClient.getMatchingRequests((Priority) any(),
-            eq("/default-rack"), (Resource) any())).thenReturn(
-        rackContainers).thenReturn(null);    
+        mockRMClient.getMatchingRequests((Priority) any(), eq("/default-rack"),
+            (Resource) any())).thenAnswer(
+        new Answer<List<? extends Collection<CookieContainerRequest>>>() {
+          @Override
+          public List<? extends Collection<CookieContainerRequest>> answer(
+              InvocationOnMock invocation) throws Throwable {
+            return rackList;
+          }
+
+        }).thenAnswer(
+        new Answer<List<? extends Collection<CookieContainerRequest>>>() {
+          @Override
+          public List<? extends Collection<CookieContainerRequest>> answer(
+              InvocationOnMock invocation) throws Throwable {
+            return emptyList;
+          }
+
+        });
     // third request matched to ANY
     when(
         mockRMClient.getMatchingRequests((Priority) any(),
-            eq(ResourceRequest.ANY), (Resource) any())).thenReturn(
-        anyContainers).thenReturn(null);
+            eq(ResourceRequest.ANY), (Resource) any())).thenAnswer(
+        new Answer<List<? extends Collection<CookieContainerRequest>>>() {
+          @Override
+          public List<? extends Collection<CookieContainerRequest>> answer(
+              InvocationOnMock invocation) throws Throwable {
+            return anyList;
+          }
+
+        }).thenAnswer(
+        new Answer<List<? extends Collection<CookieContainerRequest>>>() {
+          @Override
+          public List<? extends Collection<CookieContainerRequest>> answer(
+              InvocationOnMock invocation) throws Throwable {
+            return emptyList;
+          }
+
+        });
     scheduler.onContainersAllocated(containers);
     // first container allocated
     verify(mockApp).taskAllocated(mockTask1, mockCookie1, mockContainer1);


[36/50] [abbrv] git commit: TEZ-168. Fix synchronization issue in use of container tokens when launching a container.

Posted by ss...@apache.org.
TEZ-168. Fix synchronization issue in use of container tokens when launching a container.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/34de34b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/34de34b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/34de34b3

Branch: refs/heads/master
Commit: 34de34b31ec87e7abbdd1fabe067801a4aa1fe01
Parents: 9f0d4c3
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu May 30 13:21:01 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu May 30 13:21:01 2013 -0700

----------------------------------------------------------------------
 .../dag/app/rm/container/AMContainerHelpers.java   |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/34de34b3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 357eeda..c17f7f2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -188,7 +188,7 @@ public class AMContainerHelpers {
     // Construct the actual Container
     ContainerLaunchContext container =
         ContainerLaunchContext.newInstance(lResources, myEnv, commands,
-            myServiceData, commonContainerSpec.getTokens(), acls);
+            myServiceData, commonContainerSpec.getTokens().duplicate(), acls);
 
     return container;
   }


[05/50] [abbrv] git commit: TEZ-25: rename and cleanup: toplevel name -> DAGProtos. Job message name -> DAGPlan

Posted by ss...@apache.org.
TEZ-25: rename and cleanup: toplevel name -> DAGProtos. Job message name -> DAGPlan


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/8aa2f300
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/8aa2f300
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/8aa2f300

Branch: refs/heads/master
Commit: 8aa2f300c80c9251f39db77830dd6a4bf7003794
Parents: 45db1be
Author: mikelid <mi...@microsoft.com>
Authored: Mon May 13 14:04:15 2013 -0700
Committer: mikelid <mi...@microsoft.com>
Committed: Mon May 13 14:04:15 2013 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/tez/dag/api/DAG.java  |   15 +++++++++----
 .../org/apache/tez/dag/api/DagTypeConverters.java  |   16 +++++++++-----
 tez-dag-api/src/main/proto/DAGPlan.proto           |    4 +-
 .../java/org/apache/tez/dag/api/TestDAGPlan.java   |   15 ++++++++-----
 .../java/org/apache/tez/dag/app/DAGAppMaster.java  |   16 +++++++-------
 .../org/apache/tez/dag/app/MRRExampleHelper.java   |   10 ++++----
 .../main/java/org/apache/tez/dag/app/dag/DAG.java  |    5 +--
 .../java/org/apache/tez/dag/app/dag/Vertex.java    |    2 +-
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |   12 +++++-----
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |    2 +-
 .../apache/tez/dag/utils/TezEngineChildJVM.java    |    1 -
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |    4 +-
 12 files changed, 56 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
index efaf276..171921d 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -27,10 +27,15 @@ import java.util.Map.Entry;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.tez.dag.api.DAGPlan.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.DAGProtos.PlanKeyValuePair;
+import org.apache.tez.dag.api.DAGProtos.PlanLocalResource;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.DAGPlan.*;
-
 
 public class DAG { // FIXME rename to Topology
   List<Vertex> vertices;
@@ -88,8 +93,8 @@ public class DAG { // FIXME rename to Topology
   }
     
   // create protobuf message describing DAG
-  public JobPlan createDag(){
-    JobPlan.Builder jobBuilder = JobPlan.newBuilder();
+  public DAGPlan createDag(){
+    DAGPlan.Builder jobBuilder = DAGPlan.newBuilder();  
 
     jobBuilder.setName(this.name);
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index afbabad..c5fd0a2 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -31,15 +31,19 @@ import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.DAGPlan.EdgePlan;
-import org.apache.tez.dag.api.DAGPlan.PlanKeyValuePair;
-import org.apache.tez.dag.api.DAGPlan.PlanLocalResource;
-import org.apache.tez.dag.api.DAGPlan.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGPlan.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.DAGProtos.PlanEdgeConnectionPattern;
+import org.apache.tez.dag.api.DAGProtos.PlanEdgeSourceType;
+import org.apache.tez.dag.api.DAGProtos.PlanKeyValuePair;
+import org.apache.tez.dag.api.DAGProtos.PlanLocalResource;
+import org.apache.tez.dag.api.DAGProtos.PlanLocalResourceType;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.PlanLocalResourceVisibility;
 import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
 import org.apache.tez.dag.api.EdgeProperty.SourceType;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.DAGPlan.*;
+
 
 public class DagTypeConverters {
   

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag-api/src/main/proto/DAGPlan.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGPlan.proto b/tez-dag-api/src/main/proto/DAGPlan.proto
index 38e8971..30fc54e 100644
--- a/tez-dag-api/src/main/proto/DAGPlan.proto
+++ b/tez-dag-api/src/main/proto/DAGPlan.proto
@@ -17,7 +17,7 @@
  */
 
 option java_package = "org.apache.tez.dag.api";
-option java_outer_classname = "DAGPlan";
+option java_outer_classname = "DAGProtos";
 option java_generate_equals_and_hash = true;
 package org.apache.tez.dag.api;
 
@@ -111,7 +111,7 @@ message EdgePlan {
   required string outputClass = 7;
 }
 
-message JobPlan {
+message DAGPlan {
   required string name = 1;
   repeated VertexPlan vertex = 3;
   repeated EdgePlan edge = 4;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
index f89a73e..a041f41 100644
--- a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
+++ b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -23,8 +23,11 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 
-import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.DAGPlan.*;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -38,7 +41,7 @@ public class TestDAGPlan {
   @Test
   public void testBasicJobPlanSerde() throws IOException {
 
-    JobPlan job = JobPlan.newBuilder()
+    DAGPlan job = DAGPlan.newBuilder()
        .setName("test")
        .addVertex(
            VertexPlan.newBuilder()
@@ -46,7 +49,7 @@ public class TestDAGPlan {
              .setType(PlanVertexType.NORMAL)
              .addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("machineName").addRack("rack1").build())
              .setTaskConfig(
-                 DAGPlan.PlanTaskConfiguration.newBuilder()
+                 PlanTaskConfiguration.newBuilder()
                    .setNumTasks(2)
                    .setVirtualCores(4)
                    .setMemoryMb(1024)
@@ -67,11 +70,11 @@ public class TestDAGPlan {
      }
    }
 
-   JobPlan inJob;
+   DAGPlan inJob;
    FileInputStream inputStream;
    try {
      inputStream = new FileInputStream(file);
-     inJob = JobPlan.newBuilder().mergeFrom(inputStream).build();
+     inJob = DAGPlan.newBuilder().mergeFrom(inputStream).build();
    }
    finally {
      outStream.close();  

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 57003c1..174d210 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -63,8 +63,8 @@ import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.service.CompositeService;
 import org.apache.hadoop.yarn.service.Service;
 import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.DAGPlan.JobPlan;
-import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.app.client.ClientService;
@@ -133,7 +133,7 @@ public class DAGAppMaster extends CompositeService {
   public static final int SHUTDOWN_HOOK_PRIORITY = 30;
 
   private Clock clock;
-  private final JobPlan jobPlan;
+  private final DAGPlan jobPlan;
   private long dagsStartTime;
   private final long startTime;
   private final long appSubmitTime;
@@ -180,14 +180,14 @@ public class DAGAppMaster extends CompositeService {
 
   public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
       ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
-      long appSubmitTime, JobPlan dagPB) {
+      long appSubmitTime, DAGPlan dagPB) {
     this(applicationAttemptId, containerId, nmHost, nmPort, nmHttpPort,
         new SystemClock(), appSubmitTime, dagPB);
   }
 
   public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
       ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
-      Clock clock, long appSubmitTime, JobPlan dagPB) {
+      Clock clock, long appSubmitTime, DAGPlan dagPB) {
     super(DAGAppMaster.class.getName());
     this.jobPlan = dagPB;
     this.clock = clock;
@@ -520,7 +520,7 @@ public class DAGAppMaster extends CompositeService {
 //  }
 
   /** Create and initialize (but don't start) a single dag. */
-  protected DAG createDAG(JobPlan dagPB) {
+  protected DAG createDAG(DAGPlan dagPB) {
 
     // create single job
     DAG newDag =
@@ -957,7 +957,7 @@ public class DAGAppMaster extends CompositeService {
       String type;
       TezConfiguration conf = new TezConfiguration(new YarnConfiguration());
       
-      JobPlan jobPlan = null;
+      DAGPlan jobPlan = null;
       if (cliParser.hasOption(OPT_PREDEFINED)) {
         LOG.info("Running with PreDefined configuration");
         type = cliParser.getOptionValue(OPT_PREDEFINED, "mr");
@@ -971,7 +971,7 @@ public class DAGAppMaster extends CompositeService {
       } 
       else {
         // Read the protobuf DAG
-        JobPlan.Builder dagPlanBuilder = JobPlan.newBuilder(); 
+        DAGPlan.Builder dagPlanBuilder = DAGPlan.newBuilder(); 
         FileInputStream dagPBBinaryStream = null;
         try {
           dagPBBinaryStream = new FileInputStream(TezConfiguration.DAG_AM_PLAN_PB_BINARY);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
index b50887a..0d845b2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
@@ -17,7 +17,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.dag.api.DAGPlan.JobPlan;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.Edge;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
@@ -87,7 +87,7 @@ public class MRRExampleHelper {
  // TODO: these preconfigured jobs seem to require User and perhaps some other work.
  //       -> not tested with new DagPB system.
  
- static JobPlan createDAGConfigurationForMRR() throws IOException {
+ static DAGPlan createDAGConfigurationForMRR() throws IOException {
    org.apache.tez.dag.api.DAG dag = new org.apache.tez.dag.api.DAG();
    Vertex mapVertex = new Vertex("map",
        "org.apache.tez.mapreduce.task.InitialTask", 6);
@@ -149,12 +149,12 @@ public class MRRExampleHelper {
    dag.addConfiguration(MRJobConfig.MAP_SPECULATIVE, new Boolean(false).toString());
    dag.addConfiguration(MRJobConfig.REDUCE_SPECULATIVE, new Boolean(false).toString());
    
-   JobPlan dagPB = dag.createDag();
+   DAGPlan dagPB = dag.createDag();
    return dagPB;
  }
 
  // TODO remove once client is in place
- static JobPlan createDAGConfigurationForMR() throws IOException {
+ static DAGPlan createDAGConfigurationForMR() throws IOException {
    org.apache.tez.dag.api.DAG dag = new org.apache.tez.dag.api.DAG();
    Vertex mapVertex = new Vertex("map",
        "org.apache.tez.mapreduce.task.InitialTask", 6);
@@ -200,7 +200,7 @@ public class MRRExampleHelper {
    dag.addConfiguration(MRJobConfig.MAP_SPECULATIVE, new Boolean(false).toString());
    dag.addConfiguration(MRJobConfig.REDUCE_SPECULATIVE, new Boolean(false).toString());
    
-   JobPlan dagPB = dag.createDag();
+   DAGPlan dagPB = dag.createDag();
    
    return dagPB;
  }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index 39cfc1a..da0e38a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -24,9 +24,8 @@ import java.util.Map;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGPlan.JobPlan;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.engine.records.TezDAGID;
 import org.apache.tez.engine.records.TezVertexID;
 
@@ -60,7 +59,7 @@ public interface DAG {
   
   TezConfiguration getConf();
   
-  JobPlan getJobPlan();
+  DAGPlan getJobPlan();
   
   /**
    * @return the ACLs for this job for each type of JobACL given. 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index 8d9fbc3..1e253aa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -28,7 +28,7 @@ import org.apache.tez.common.InputSpec;
 import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.EdgeProperty;
-import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
 import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
 import org.apache.tez.engine.records.TezTaskID;
 import org.apache.tez.engine.records.TezVertexID;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 2e20692..cc3170e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -52,9 +52,9 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGPlan.EdgePlan;
-import org.apache.tez.dag.api.DAGPlan.JobPlan;
-import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint;
@@ -132,7 +132,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   private TezCounters fullCounters = null;
 
   public final TezConfiguration conf;
-  private final JobPlan jobPlan;
+  private final DAGPlan jobPlan;
 
   //fields initialized in init
   private FileSystem fs;
@@ -311,7 +311,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
 
   public DAGImpl(TezDAGID dagId, ApplicationAttemptId applicationAttemptId,
       TezConfiguration conf,
-      JobPlan jobPlan,
+      DAGPlan jobPlan,
       EventHandler eventHandler,
       TaskAttemptListener taskAttemptListener,
       JobTokenSecretManager jobTokenSecretManager,
@@ -373,7 +373,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   }
   
   @Override
-  public JobPlan getJobPlan() {
+  public DAGPlan getJobPlan() {
     return jobPlan;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index a423db1..b9e911c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -55,7 +55,7 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.tez.common.InputSpec;
 import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
index b3db770..fee0846 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.app.rm.container.AMContainerHelpers;
 import org.apache.tez.engine.records.TezVertexID;
 
 public class TezEngineChildJVM {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 9042846..c0c9923 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -99,7 +99,7 @@ import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DAGPlan.JobPlan;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.Edge;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;
@@ -804,7 +804,7 @@ public class YARNRunner implements ClientProtocol {
     setDAGParamsFromMRConf(dag);
 
     // emit protobuf DAG file style
-    JobPlan dagPB = dag.createDag();
+    DAGPlan dagPB = dag.createDag();
     FSDataOutputStream dagPBOutBinaryStream = null;
     FSDataOutputStream dagPBOutTextStream = null;
     Path binaryPath =  new Path(jobSubmitDir, TezConfiguration.DAG_AM_PLAN_PB_BINARY);


[15/50] [abbrv] git commit: TEZ-9. Add support for client api and records. cleaned up packages (bikas)

Posted by ss...@apache.org.
TEZ-9. Add support for client api and records. cleaned up packages (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/de5ee038
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/de5ee038
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/de5ee038

Branch: refs/heads/master
Commit: de5ee0382085e91e5644d881182309bc6858e9ed
Parents: cee79f6
Author: Bikas Saha <bi...@apache.org>
Authored: Fri May 17 15:59:37 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Fri May 17 15:59:37 2013 -0700

----------------------------------------------------------------------
 .../java/org/apache/tez/dag/records/TezDAGID.java  |   14 ++
 tez-dag-api/pom.xml                                |    3 +-
 .../src/main/java/org/apache/tez/dag/api/DAG.java  |   16 +-
 .../org/apache/tez/dag/api/DagTypeConverters.java  |   18 +-
 .../org/apache/tez/dag/api/client/DAGClient.java   |   46 ++++
 .../org/apache/tez/dag/api/client/DAGStatus.java   |   95 ++++++++
 .../org/apache/tez/dag/api/client/Progress.java    |   51 +++++
 .../apache/tez/dag/api/client/VertexStatus.java    |   75 +++++++
 .../client/rpc/DAGClientAMProtocolBlockingPB.java  |   30 +++
 .../tez/dag/api/client/rpc/DAGClientRPCImpl.java   |  103 +++++++++
 tez-dag-api/src/main/proto/DAGApiRecords.proto     |  167 +++++++++++++++
 .../src/main/proto/DAGClientAMProtocol.proto       |   56 +++++
 tez-dag-api/src/main/proto/DAGPlan.proto           |  119 ----------
 .../java/org/apache/tez/dag/api/TestDAGPlan.java   |   10 +-
 .../hadoop/mapred/MRVertexOutputCommitter.java     |    8 +-
 .../tez/dag/api/client/DAGStatusBuilder.java       |   86 ++++++++
 .../apache/tez/dag/api/client/ProgressBuilder.java |   58 +++++
 .../tez/dag/api/client/VertexStatusBuilder.java    |   78 +++++++
 .../tez/dag/api/client/impl/TezBuilderUtils.java   |   54 -----
 .../apache/tez/dag/api/committer/DAGStatus.java    |   39 ++++
 .../api/committer/NullVertexOutputCommitter.java   |   47 ++++
 .../tez/dag/api/committer/VertexContext.java       |   37 ++++
 .../dag/api/committer/VertexOutputCommitter.java   |   69 ++++++
 .../apache/tez/dag/api/committer/VertexStatus.java |   51 +++++
 .../org/apache/tez/dag/api/impl/DAGStatus.java     |   39 ----
 .../dag/api/impl/NullVertexOutputCommitter.java    |   47 ----
 .../org/apache/tez/dag/api/impl/VertexContext.java |   37 ----
 .../tez/dag/api/impl/VertexOutputCommitter.java    |   69 ------
 .../org/apache/tez/dag/api/impl/VertexStatus.java  |   51 -----
 .../org/apache/tez/dag/api/oldrecords/AMInfo.java  |   38 ++++
 .../tez/dag/api/oldrecords/TaskAttemptReport.java  |   66 ++++++
 .../tez/dag/api/oldrecords/TaskAttemptState.java   |   29 +++
 .../apache/tez/dag/api/oldrecords/TaskReport.java  |   63 ++++++
 .../apache/tez/dag/api/oldrecords/TaskState.java   |   23 ++
 .../org/apache/tez/dag/api/records/AMInfo.java     |   38 ----
 .../tez/dag/api/records/TaskAttemptReport.java     |   66 ------
 .../tez/dag/api/records/TaskAttemptState.java      |   29 ---
 .../org/apache/tez/dag/api/records/TaskReport.java |   63 ------
 .../org/apache/tez/dag/api/records/TaskState.java  |   23 --
 .../java/org/apache/tez/dag/app/DAGAppMaster.java  |   41 ++++-
 .../org/apache/tez/dag/app/MRRExampleHelper.java   |    2 +-
 .../main/java/org/apache/tez/dag/app/dag/DAG.java  |    7 +-
 .../java/org/apache/tez/dag/app/dag/DAGReport.java |    2 +-
 .../main/java/org/apache/tez/dag/app/dag/Task.java |    4 +-
 .../org/apache/tez/dag/app/dag/TaskAttempt.java    |    4 +-
 .../java/org/apache/tez/dag/app/dag/Vertex.java    |    6 +-
 .../dag/event/TaskAttemptEventStatusUpdate.java    |    2 +-
 .../app/dag/event/VertexEventTaskCompleted.java    |    2 +-
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |   58 +++++-
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java      |    6 +-
 .../org/apache/tez/dag/app/dag/impl/TaskImpl.java  |    6 +-
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |   51 ++++-
 .../tez/dag/app/rm/AMSchedulerEventTAEnded.java    |    2 +-
 .../tez/dag/app/rm/TaskSchedulerEventHandler.java  |    2 +-
 .../tez/dag/app/speculate/DefaultSpeculator.java   |    4 +-
 .../app/speculate/LegacyTaskRuntimeEstimator.java  |    2 +-
 .../tez/dag/app/speculate/StartEndTimesBase.java   |    2 +-
 .../tez/dag/history/events/DAGFinishedEvent.java   |    2 +-
 .../history/events/TaskAttemptFinishedEvent.java   |    2 +-
 .../tez/dag/history/events/TaskFinishedEvent.java  |    2 +-
 .../dag/history/events/VertexFinishedEvent.java    |    2 +-
 .../org/apache/tez/dag/utils/TezBuilderUtils.java  |   54 +++++
 .../tez/dag/app/dag/impl/TestTaskAttempt.java      |    2 +-
 .../apache/tez/dag/app/dag/impl/TestTaskImpl.java  |    4 +-
 .../tez/dag/app/dag/impl/TestVertexImpl.java       |   26 ++--
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |    2 +-
 66 files changed, 1587 insertions(+), 723 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
index e977dc0..733a908 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
@@ -100,6 +100,20 @@ public class TezDAGID extends TezID {
   public String toString() {
     return appendTo(new StringBuilder(DAG)).toString();
   }
+  
+  public static TezDAGID fromString(String dagId) {
+    try {
+      String[] split = dagId.split("_");
+      String rmId = split[1];
+      int appId = Integer.parseInt(split[2]);
+      int id;
+      id = idFormat.parse(split[3]).intValue();
+      return new TezDAGID(rmId, appId, id);
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+    return null;
+  }
 
   @Override
   public void readFields(DataInput in) throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag-api/pom.xml b/tez-dag-api/pom.xml
index 2b6aebb..74b1284 100644
--- a/tez-dag-api/pom.xml
+++ b/tez-dag-api/pom.xml
@@ -64,7 +64,8 @@
               <source>
                 <directory>${basedir}/src/main/proto</directory>
                 <includes>
-                  <include>DAGPlan.proto</include>
+                  <include>DAGApiRecords.proto</include>
+                  <include>DAGClientAMProtocol.proto</include>
                 </includes>
               </source>
               <output>${project.build.directory}/generated-sources/java</output>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 300ef31..7fa764a 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -28,14 +28,14 @@ import java.util.Stack;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.EdgePlan;
-import org.apache.tez.dag.api.DAGProtos.PlanKeyValuePair;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResource;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
-import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResource;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
 
 public class DAG { // FIXME rename to Topology

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index c5fd0a2..ba42e23 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -31,15 +31,15 @@ import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.DAGProtos.EdgePlan;
-import org.apache.tez.dag.api.DAGProtos.PlanEdgeConnectionPattern;
-import org.apache.tez.dag.api.DAGProtos.PlanEdgeSourceType;
-import org.apache.tez.dag.api.DAGProtos.PlanKeyValuePair;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResource;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResourceType;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResourceVisibility;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeConnectionPattern;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeSourceType;
+import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResource;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResourceType;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResourceVisibility;
 import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
 import org.apache.tez.dag.api.EdgeProperty.SourceType;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
new file mode 100644
index 0000000..477b017
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
@@ -0,0 +1,46 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.TezException;
+
+/*
+ * Inteface class for monitoring the <code>DAG</code> running in a Tez DAG
+ * Application Master.
+ */
+public interface DAGClient {
+  
+  /**
+   * Return the identifiers for all DAG's
+   */
+  List<String> getAllDAGs() throws TezException;
+  
+  /**
+   * Get the status of a DAG
+   */
+  DAGStatus getDAGStatus(String dagId) throws TezException;
+  
+  /**
+   * Get the status of a Vertex of a DAG 
+   */
+  VertexStatus getVertexStatus(String dagId, 
+                               String vertexName) throws TezException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
new file mode 100644
index 0000000..597789b
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
@@ -0,0 +1,95 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusProtoOrBuilder;
+import org.apache.tez.dag.api.records.DAGProtos.StringProgressPairProto;
+import org.apache.tez.dag.api.TezException;
+
+public class DAGStatus {
+
+  public enum State {
+    SUBMITTED,
+    INITING,
+    RUNNING,
+    SUCCEEDED,
+    KILLED,
+    FAILED,
+    ERROR
+  };
+  
+  DAGStatusProtoOrBuilder proxy = null;
+  Progress progress = null;
+  Map<String, Progress> vertexProgress = null;
+  
+  public DAGStatus(DAGStatusProtoOrBuilder proxy) {
+    this.proxy = proxy;
+  }
+  
+  public State getState() {
+    switch(proxy.getState()) {
+    case DAG_SUBMITTED:
+      return DAGStatus.State.SUBMITTED;
+    case DAG_INITING:
+      return DAGStatus.State.INITING;
+    case DAG_RUNNING:
+      return DAGStatus.State.RUNNING;
+    case DAG_SUCCEEDED:
+      return DAGStatus.State.SUCCEEDED;
+    case DAG_FAILED:
+      return DAGStatus.State.FAILED;
+    case DAG_KILLED:
+      return DAGStatus.State.KILLED;
+    case DAG_ERROR:
+      return DAGStatus.State.ERROR;
+    default:
+      throw new TezException("Unsupported value for DAGStatus.State : " + 
+                              proxy.getState());
+    }
+  }
+
+  public List<String> getDiagnostics() {
+    return proxy.getDiagnosticsList();
+  }
+
+  public Progress getDAGProgress() {
+    if(progress == null && proxy.hasDAGProgress()) {
+      progress = new Progress(proxy.getDAGProgress());
+    }
+    return progress;
+  }
+
+  public Map<String, Progress> getVertexProgress() {
+    if(vertexProgress == null) {
+      if(proxy.getVertexProgressList() != null) {
+        List<StringProgressPairProto> kvList = proxy.getVertexProgressList();
+        vertexProgress = new HashMap<String, Progress>(kvList.size());        
+        for(StringProgressPairProto kv : kvList){
+          vertexProgress.put(kv.getKey(), new Progress(kv.getProgress()));
+        }
+      }
+    }
+    return vertexProgress;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
new file mode 100644
index 0000000..7158e30
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import org.apache.tez.dag.api.records.DAGProtos.ProgressProtoOrBuilder;
+
+public class Progress {
+  
+  ProgressProtoOrBuilder proxy = null;
+  
+  Progress(ProgressProtoOrBuilder proxy) {
+    this.proxy = proxy;
+  }
+  
+  public int getTotalTaskCount() {
+    return proxy.getTotalTaskCount();
+  }
+
+  public int getSucceededTaskCount() {
+    return proxy.getSucceededTaskCount();
+  }
+
+  public int getRunningTaskCount() {
+    return proxy.getRunningTaskCount();
+  }
+
+  public int getFailedTaskCount() {
+    return proxy.getFailedTaskCount();
+  }
+
+  public int getKilledTaskCount() {
+    return proxy.getKilledTaskCount();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
new file mode 100644
index 0000000..33b4de3
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
@@ -0,0 +1,75 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProtoOrBuilder;
+import org.apache.tez.dag.api.TezException;
+
+public class VertexStatus {
+  
+  public enum State {
+    INITED,
+    RUNNING,
+    SUCCEEDED,
+    KILLED,
+    FAILED,
+    ERROR
+  };
+  
+  VertexStatusProtoOrBuilder proxy = null;
+  Progress progress = null;
+  
+  public VertexStatus(VertexStatusProtoOrBuilder proxy) {
+    this.proxy = proxy;
+  }
+
+  public State getState() {
+    switch(proxy.getState()) {
+    case VERTEX_INITED:
+      return VertexStatus.State.INITED;
+    case VERTEX_RUNNING:
+      return VertexStatus.State.RUNNING;
+    case VERTEX_SUCCEEDED:
+      return VertexStatus.State.SUCCEEDED;
+    case VERTEX_FAILED:
+      return VertexStatus.State.FAILED;
+    case VERTEX_KILLED:
+      return VertexStatus.State.KILLED;
+    case VERTEX_ERROR:
+      return VertexStatus.State.ERROR;
+    default:
+      throw new TezException("Unsupported value for VertexStatus.State : " + 
+                              proxy.getState());
+    }    
+  }
+
+  public List<String> getDiagnostics() {
+    return proxy.getDiagnosticsList();
+  }
+
+  public Progress getProgress() {
+    if(progress == null && proxy.hasProgress()) {
+      progress = new Progress(proxy.getProgress());
+    }
+    return progress;    
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
new file mode 100644
index 0000000..a1ee18f
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.rpc;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.DAGClientAMProtocol;
+
+@ProtocolInfo(
+    protocolName = "org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolPB",
+    protocolVersion = 1)
+public interface DAGClientAMProtocolBlockingPB 
+                              extends DAGClientAMProtocol.BlockingInterface {
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
new file mode 100644
index 0000000..9238f63
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.rpc;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAllDAGsRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetDAGStatusRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetVertexStatusRequestProto;
+
+import com.google.protobuf.ServiceException;
+
+public class DAGClientRPCImpl implements DAGClient, Closeable {
+
+  private DAGClientAMProtocolBlockingPB proxy = null;
+  
+  public DAGClientRPCImpl(long clientVersion, 
+                          InetSocketAddress addr,
+                          TezConfiguration conf) throws IOException {
+    RPC.setProtocolEngine(conf, 
+                          DAGClientAMProtocolBlockingPB.class, 
+                          ProtobufRpcEngine.class);
+    proxy =
+        (DAGClientAMProtocolBlockingPB) 
+        RPC.getProxy(DAGClientAMProtocolBlockingPB.class, 
+                     clientVersion,
+                     addr, 
+                     conf);
+  }
+  
+  @Override
+  public List<String> getAllDAGs() throws TezException {
+    GetAllDAGsRequestProto requestProto = 
+        GetAllDAGsRequestProto.newBuilder().build();
+    try {
+      return proxy.getAllDAGs(null, requestProto).getDagIdList();
+    } catch (ServiceException e) {
+      throw new TezException(e);
+    }
+  }
+
+  @Override
+  public DAGStatus getDAGStatus(String dagId) throws TezException {
+    GetDAGStatusRequestProto requestProto = 
+        GetDAGStatusRequestProto.newBuilder().setDagId(dagId).build();
+    
+    try {
+      return new DAGStatus(
+                 proxy.getDAGStatus(null, requestProto).getDagStatus());
+    } catch (ServiceException e) {
+      throw new TezException(e);
+    }
+  }
+
+  @Override
+  public VertexStatus getVertexStatus(String dagId, String vertexName)
+      throws TezException {
+    GetVertexStatusRequestProto requestProto = 
+        GetVertexStatusRequestProto.newBuilder().
+                        setDagId(dagId).setVertexName(vertexName).build();
+    
+    try {
+      return new VertexStatus(
+                 proxy.getVertexStatus(null, requestProto).getVertexStatus());
+    } catch (ServiceException e) {
+      throw new TezException(e);
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (this.proxy != null) {
+      RPC.stopProxy(this.proxy);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/proto/DAGApiRecords.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGApiRecords.proto b/tez-dag-api/src/main/proto/DAGApiRecords.proto
new file mode 100644
index 0000000..242715b
--- /dev/null
+++ b/tez-dag-api/src/main/proto/DAGApiRecords.proto
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tez.dag.api.records";
+option java_outer_classname = "DAGProtos";
+option java_generate_equals_and_hash = true;
+
+// DAG plan messages
+
+// Many of these types have a dual in the Tez-api.  To reduce confusion, these types have prefix or suffix 
+// of "Plan" to indicate they are to be used in the dag-plan.
+// The big types use a suffix:  JobPlan, VertexPlan, EdgePlan 
+//   --> these get more direct use in the runtime and the naming is natural.
+// The enums and utility types use prefix: PlanVertexType, PlanEdgeConnectionPaatern, etc
+//   --> there is not great naming choice for these that avoids ambiguity, but this one seems acceptable. 
+
+enum PlanVertexType {
+  INPUT = 0;
+  NORMAL = 1;
+  OUTPUT = 2;
+}
+
+enum PlanEdgeConnectionPattern {
+  ONE_TO_ONE = 0;
+  ONE_TO_ALL = 1;
+  BIPARTITE = 2;
+}
+
+enum PlanEdgeSourceType {
+  STABLE = 0;
+  STABLE_PERSISTED = 1;
+  STREAMING = 2;
+}
+
+message PlanKeyValuePair {
+  required string key = 1;
+  required string value = 2;
+}
+
+enum PlanLocalResourceType {
+  FILE = 0;
+  ARCHIVE = 1;
+  PATTERN = 2;
+}
+
+enum PlanLocalResourceVisibility {
+  PUBLIC = 0;
+  PRIVATE = 1;
+  APPLICATION = 2;
+}
+
+message PlanLocalResource {
+  required string name = 1;
+  required string uri = 2;
+  required int64 size = 3;
+  required int64 timeStamp = 4;
+  required PlanLocalResourceType type = 5;
+  required PlanLocalResourceVisibility visibility = 6;
+  optional string pattern = 7; // only used if type=PATTERN
+}
+
+// Each taskLocationHint represents a single split in in the input.
+// It is the list of [{rack,machines}] that host a replica of each particular split.
+// For now it is represented as pair-of-arrays rather than array-of-pairs.
+message PlanTaskLocationHint {
+  repeated string rack = 1;
+  repeated string host = 2;
+}
+
+message PlanTaskConfiguration {
+  required int32 numTasks = 1;
+  required int32 memoryMb = 2;
+  required int32 virtualCores = 3;
+  required string javaOpts = 4;
+  required string taskModule = 5;
+  repeated PlanLocalResource localResource = 6;
+  repeated PlanKeyValuePair environmentSetting = 8;  
+}
+
+message VertexPlan {
+  required string name = 1;
+  required PlanVertexType type = 2;
+  optional string processorName = 3;
+  required PlanTaskConfiguration taskConfig = 4;
+  repeated PlanTaskLocationHint taskLocationHint = 7;
+  repeated string inEdgeId = 8;
+  repeated string outEdgeId = 9;
+}
+
+message EdgePlan {
+  required string id = 1;
+  required string inputVertexName = 2;
+  required string outputVertexName = 3;
+  required PlanEdgeConnectionPattern connectionPattern = 4;
+  required PlanEdgeSourceType sourceType = 5;
+  required string inputClass = 6;
+  required string outputClass = 7;
+}
+
+message DAGPlan {
+  required string name = 1;
+  repeated VertexPlan vertex = 2;
+  repeated EdgePlan edge = 3;
+  repeated PlanKeyValuePair jobSetting = 4;
+}
+
+
+// DAG monitoring messages
+message ProgressProto {
+  optional int32 totalTaskCount = 1;
+  optional int32 succeededTaskCount = 2;
+  optional int32 runningTaskCount = 3;
+  optional int32 failedTaskCount = 4;
+  optional int32 killedTaskCount = 5;
+}
+
+enum VertexStatusStateProto {
+  VERTEX_INITED = 0;
+  VERTEX_RUNNING = 1;
+  VERTEX_SUCCEEDED = 2;
+  VERTEX_KILLED = 3;
+  VERTEX_FAILED = 4;
+  VERTEX_ERROR = 5;
+}
+
+message VertexStatusProto {
+  optional VertexStatusStateProto state = 1;
+  repeated string diagnostics = 2;
+  optional ProgressProto progress = 3;
+}
+
+enum DAGStatusStateProto {
+  DAG_SUBMITTED = 0;
+  DAG_INITING = 1;
+  DAG_RUNNING = 2;
+  DAG_SUCCEEDED = 3;
+  DAG_KILLED = 4;
+  DAG_FAILED = 5;
+  DAG_ERROR = 6;
+}
+
+message StringProgressPairProto {
+  required string key = 1;
+  required ProgressProto progress = 2;
+}
+
+message DAGStatusProto {
+  optional DAGStatusStateProto state = 1;
+  repeated string diagnostics = 2;
+  optional ProgressProto DAGProgress = 3;
+  repeated StringProgressPairProto vertexProgress = 4;  
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto b/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto
new file mode 100644
index 0000000..18247a1
--- /dev/null
+++ b/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tez.dag.api.client.rpc";
+option java_outer_classname = "DAGClientAMProtocolRPC";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+//import "DAGClientAMProtocolRecords.proto";
+
+import "DAGApiRecords.proto";
+
+message GetAllDAGsRequestProto {
+}
+
+message GetAllDAGsResponseProto {
+  repeated string dagId = 1;
+}
+
+message GetDAGStatusRequestProto {
+  optional string dagId = 1;
+}
+
+message GetDAGStatusResponseProto {
+  optional DAGStatusProto dagStatus = 1; 
+}
+
+message GetVertexStatusRequestProto {
+  optional string dagId = 1;
+  optional string vertexName = 2;
+}
+
+message GetVertexStatusResponseProto {
+  optional VertexStatusProto vertexStatus = 1;
+}
+
+service DAGClientAMProtocol {
+  rpc getAllDAGs (GetAllDAGsRequestProto) returns (GetAllDAGsResponseProto);
+  rpc getDAGStatus (GetDAGStatusRequestProto) returns (GetDAGStatusResponseProto);
+  rpc getVertexStatus (GetVertexStatusRequestProto) returns (GetVertexStatusResponseProto);
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/proto/DAGPlan.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGPlan.proto b/tez-dag-api/src/main/proto/DAGPlan.proto
deleted file mode 100644
index 30fc54e..0000000
--- a/tez-dag-api/src/main/proto/DAGPlan.proto
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.tez.dag.api";
-option java_outer_classname = "DAGProtos";
-option java_generate_equals_and_hash = true;
-package org.apache.tez.dag.api;
-
-// Many of these types have a dual in the Tez-api.  To reduce confusion, these types have prefix or suffix 
-// of "Plan" to indicate they are to be used in the dag-plan.
-// The big types use a suffix:  JobPlan, VertexPlan, EdgePlan 
-//   --> these get more direct use in the runtime and the naming is natural.
-// The enums and utility types use prefix: PlanVertexType, PlanEdgeConnectionPaatern, etc
-//   --> there is not great naming choice for these that avoids ambiguity, but this one seems acceptable. 
-
-enum PlanVertexType {
-  INPUT = 0;
-  NORMAL = 1;
-  OUTPUT = 2;
-}
-
-enum PlanEdgeConnectionPattern {
-  ONE_TO_ONE = 0;
-  ONE_TO_ALL = 1;
-  BIPARTITE = 2;
-}
-
-enum PlanEdgeSourceType {
-  STABLE = 0;
-  STABLE_PERSISTED = 1;
-  STREAMING = 2;
-}
-
-message PlanKeyValuePair {
-  required string key = 1;
-  required string value = 2;
-}
-
-enum PlanLocalResourceType {
-  FILE = 0;
-  ARCHIVE = 1;
-  PATTERN = 2;
-}
-
-enum PlanLocalResourceVisibility {
-  PUBLIC = 0;
-  PRIVATE = 1;
-  APPLICATION = 2;
-}
-
-message PlanLocalResource {
-  required string name = 1;
-  required string uri = 2;
-  required int64 size = 3;
-  required int64 timeStamp = 4;
-  required PlanLocalResourceType type = 5;
-  required PlanLocalResourceVisibility visibility = 6;
-  optional string pattern = 7; // only used if type=PATTERN
-}
-
-// Each taskLocationHint represents a single split in in the input.
-// It is the list of [{rack,machines}] that host a replica of each particular split.
-// For now it is represented as pair-of-arrays rather than array-of-pairs.
-message PlanTaskLocationHint {
-  repeated string rack = 1;
-  repeated string host = 2;
-}
-
-message PlanTaskConfiguration {
-  required int32 numTasks = 1;
-  required int32 memoryMb = 2;
-  required int32 virtualCores = 3;
-  required string javaOpts = 4;
-  required string taskModule = 5;
-  repeated PlanLocalResource localResource = 6;
-  repeated PlanKeyValuePair environmentSetting = 8;  
-}
-
-message VertexPlan {
-  required string name = 1;
-  required PlanVertexType type = 2;
-  optional string processorName = 3;
-  required PlanTaskConfiguration taskConfig = 4;
-  repeated PlanTaskLocationHint taskLocationHint = 7;
-  repeated string inEdgeId = 8;
-  repeated string outEdgeId = 9;
-}
-
-message EdgePlan {
-  required string id = 1;
-  required string inputVertexName = 2;
-  required string outputVertexName = 3;
-  required PlanEdgeConnectionPattern connectionPattern = 4;
-  required PlanEdgeSourceType sourceType = 5;
-  required string inputClass = 6;
-  required string outputClass = 7;
-}
-
-message DAGPlan {
-  required string name = 1;
-  repeated VertexPlan vertex = 3;
-  repeated EdgePlan edge = 4;
-  repeated PlanKeyValuePair jobSetting = 5;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
index a041f41..6dca610 100644
--- a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
+++ b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -23,11 +23,11 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
-import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
index a3b72c7..7c886d6 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
@@ -33,12 +33,12 @@ import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.YarnException;
-import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
-import org.apache.tez.dag.api.impl.VertexContext;
-import org.apache.tez.dag.api.impl.VertexOutputCommitter;
-import org.apache.tez.dag.api.impl.VertexStatus;
+import org.apache.tez.dag.api.committer.VertexContext;
+import org.apache.tez.dag.api.committer.VertexOutputCommitter;
+import org.apache.tez.dag.api.committer.VertexStatus;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.utils.TezBuilderUtils;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 public class MRVertexOutputCommitter extends VertexOutputCommitter {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
new file mode 100644
index 0000000..553347f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
@@ -0,0 +1,86 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusStateProto;
+import org.apache.tez.dag.api.records.DAGProtos.StringProgressPairProto;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusProto;
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusProto.Builder;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.app.dag.DAGState;
+
+public class DAGStatusBuilder extends DAGStatus {
+
+  public DAGStatusBuilder() {
+    super(DAGStatusProto.newBuilder());
+  }
+  
+  public void setState(DAGState state) {
+    getBuilder().setState(getProtoState(state));
+  }
+  
+  public void setDiagnostics(List<String> diagnostics) {
+    Builder builder = getBuilder();
+    builder.clearDiagnostics();
+    builder.addAllDiagnostics(diagnostics);
+  }
+  
+  public void setDAGProgress(ProgressBuilder progress) {
+    getBuilder().setDAGProgress(progress.getProto());
+  }
+  
+  public void addVertexProgress(String name, ProgressBuilder progress) {
+    StringProgressPairProto.Builder builder = StringProgressPairProto.newBuilder();
+    builder.setKey(name);
+    builder.setProgress(progress.getProto());
+    getBuilder().addVertexProgress(builder.build());
+  }
+  
+  public DAGStatusProto getProto() {
+    return getBuilder().build();
+  }
+  
+  private DAGStatusStateProto getProtoState(DAGState state) {
+    switch(state) {
+    case NEW:
+    case INITED:
+      return DAGStatusStateProto.DAG_INITING;
+    case RUNNING:
+      return DAGStatusStateProto.DAG_RUNNING;
+    case SUCCEEDED:
+      return DAGStatusStateProto.DAG_SUCCEEDED;
+    case FAILED:
+      return DAGStatusStateProto.DAG_FAILED;
+    case KILLED:
+    case KILL_WAIT:
+      return DAGStatusStateProto.DAG_KILLED;
+    case ERROR:
+      return DAGStatusStateProto.DAG_ERROR;
+    default:
+      throw new TezException("Unsupported value for DAGState : " + state);
+    }
+  }
+  
+  private DAGStatusProto.Builder getBuilder() {
+    return (Builder) this.proxy;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java
new file mode 100644
index 0000000..6cedb3f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java
@@ -0,0 +1,58 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import org.apache.tez.dag.api.records.DAGProtos.ProgressProto;
+import org.apache.tez.dag.api.records.DAGProtos.ProgressProto.Builder;
+import org.apache.tez.dag.api.client.Progress;
+
+public class ProgressBuilder extends Progress {
+
+  public ProgressBuilder() {
+    super(ProgressProto.newBuilder());
+  }
+  
+  public ProgressProto getProto() {
+    return getBuilder().build();
+  }
+  
+  public void setTotalTaskCount(int count) {
+    getBuilder().setTotalTaskCount(count);
+  }
+  
+  public void setSucceededTaskCount(int count) {
+    getBuilder().setSucceededTaskCount(count);
+  }
+  
+  public void setRunningTaskCount(int count) {
+    getBuilder().setRunningTaskCount(count);
+  }
+  
+  public void setFailedTaskCount(int count) {
+    getBuilder().setFailedTaskCount(count);
+  }
+  
+  public void setKilledTaskCount(int count) {
+    getBuilder().setKilledTaskCount(count);
+  }
+  
+  private ProgressProto.Builder getBuilder() {
+    return (Builder) this.proxy;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
new file mode 100644
index 0000000..91e3e83
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
@@ -0,0 +1,78 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProto;
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProto.Builder;
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusStateProto;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.app.dag.VertexState;
+
+public class VertexStatusBuilder extends VertexStatus {
+
+  public VertexStatusBuilder() {
+    super(VertexStatusProto.newBuilder());
+  }
+  
+  public void setState(VertexState state) {
+    getBuilder().setState(getProtoState(state));
+  }
+  
+  public void setDiagnostics(List<String> diagnostics) {
+    Builder builder = getBuilder();
+    builder.clearDiagnostics();
+    builder.addAllDiagnostics(diagnostics);
+  }
+  
+  public void setProgress(ProgressBuilder progress) {
+    getBuilder().setProgress(progress.getProto());
+  }
+  
+  public VertexStatusProto getProto() {
+    return getBuilder().build();
+  }
+  
+  private VertexStatusStateProto getProtoState(VertexState state) {
+    switch(state) {
+    case NEW:
+    case INITED:
+      return VertexStatusStateProto.VERTEX_INITED;
+    case RUNNING:
+      return VertexStatusStateProto.VERTEX_RUNNING;
+    case SUCCEEDED:
+      return VertexStatusStateProto.VERTEX_SUCCEEDED;
+    case FAILED:
+      return VertexStatusStateProto.VERTEX_FAILED;
+    case KILLED:
+    case KILL_WAIT:
+      return VertexStatusStateProto.VERTEX_KILLED;
+    case ERROR:
+      return VertexStatusStateProto.VERTEX_ERROR;
+    default:
+      throw new TezException("Unsupported value for VertexState : " + state);
+    }
+  }
+  
+  private VertexStatusProto.Builder getBuilder() {
+    return (Builder) this.proxy;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
deleted file mode 100644
index 5f36a58..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.client.impl;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.dag.api.records.AMInfo;
-import org.apache.tez.dag.app.dag.DAGReport;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-import org.apache.tez.dag.records.TezVertexID;
-
-public class TezBuilderUtils {
-
-  public static TezVertexID newVertexID(TezDAGID dagId, int vertexId) {
-    return new TezVertexID(dagId, vertexId);
-  }
-
-  public static TezTaskAttemptID newTaskAttemptId(TezTaskID taskId, int id) {
-    return new TezTaskAttemptID(taskId, id);
-  }
-  
-  public static DAGReport newDAGReport() {
-    return null;
-  }
-
-  public static AMInfo newAMInfo(ApplicationAttemptId appAttemptID, 
-      long startTime, ContainerId containerID, String nmHost, 
-      int nmPort, int nmHttpPort) {
-    return null;
-  }
-
-  public static TezTaskID newTaskId(TezDAGID dagId, int vertexId, int taskId) {
-    return new TezTaskID(newVertexID(dagId, vertexId), taskId);
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
new file mode 100644
index 0000000..587e18f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
@@ -0,0 +1,39 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.committer;
+
+public class DAGStatus {
+  public static enum State {
+    RUNNING(1),
+    SUCCEEDED(2),
+    FAILED(3),
+    PREP(4),
+    KILLED(5);
+    
+    int value;
+    
+    State(int value) {
+      this.value = value;
+    }
+    
+    public int getValue() {
+      return value; 
+    }
+    
+  };
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
new file mode 100644
index 0000000..c284ab5
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.committer;
+
+import java.io.IOException;
+
+import org.apache.tez.dag.api.committer.VertexStatus.State;
+
+public class NullVertexOutputCommitter extends VertexOutputCommitter {
+
+  @Override
+  public void init(VertexContext context) throws IOException {
+    // Nothing to do
+  }
+
+  @Override
+  public void setupVertex() throws IOException {
+    // Nothing to do
+  }
+
+  @Override
+  public void commitVertex() throws IOException {
+    // Nothing to do
+  }
+
+  @Override
+  public void abortVertex(State finalState) throws IOException {
+    // Nothing to do
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
new file mode 100644
index 0000000..14880d2
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.committer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
+
+public interface VertexContext {
+
+  public Configuration getConf();
+
+  public TezDAGID getDAGId();
+  
+  // TODO Get rid of this as part of VertexContext cleanup.
+  public ApplicationAttemptId getApplicationAttemptId();
+
+  public TezVertexID getVertexId();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
new file mode 100644
index 0000000..b814d0e
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.committer;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Public
+@Unstable
+public abstract class VertexOutputCommitter {
+
+  /**
+   * Setup up the vertex output committer.
+   *
+   * @param context Context of the vertex whose output is being written.
+   * @throws IOException
+   */
+  public abstract void init(VertexContext context) throws IOException;
+
+  /**
+   * For the framework to setup the vertex output during initialization. This is
+   * called from the application master process for the vertex. This will be
+   * called multiple times, once per dag attempt for each vertex.
+   *
+   * @throws IOException if setup fails
+   */
+  public abstract void setupVertex() throws IOException;
+
+  /**
+   * For committing vertex's output after successful vertex completion.
+   * Note that this is invoked for vertices with a successful final state.
+   * This is called from the application master process for the entire vertex.
+   * This is guaranteed to only be called once.
+   * If it throws an exception the entire vertex will fail.
+   *
+   * @throws IOException
+   */
+  public abstract void commitVertex() throws IOException;
+
+  /**
+   * For aborting an unsuccessful vertex's output. Note that this is invoked for
+   * vertices with a final failed state. This is called from the application
+   * master process for the entire vertex. This may be called multiple times.
+   *
+   * @param state final runstate of the vertex
+   * @throws IOException
+   */
+  public abstract void abortVertex(VertexStatus.State finalState)
+      throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
new file mode 100644
index 0000000..a78027a
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.committer;
+
+public class VertexStatus {
+
+  public static enum State {
+    RUNNING(1),
+    SUCCEEDED(2),
+    FAILED(3),
+    PREP(4), // TODO change to INITING
+    KILLED(5);
+
+    int value;
+
+    State(int value) {
+      this.value = value;
+    }
+
+    public int getValue() {
+      return value;
+    }
+
+  };
+
+  private State runState;
+
+  public synchronized State getState() {
+    return runState;
+  }
+
+  public synchronized void setState(State state) {
+    this.runState = state;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
deleted file mode 100644
index 6ede497..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.impl;
-
-public class DAGStatus {
-  public static enum State {
-    RUNNING(1),
-    SUCCEEDED(2),
-    FAILED(3),
-    PREP(4),
-    KILLED(5);
-    
-    int value;
-    
-    State(int value) {
-      this.value = value;
-    }
-    
-    public int getValue() {
-      return value; 
-    }
-    
-  };
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
deleted file mode 100644
index 102bdec..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.impl;
-
-import java.io.IOException;
-
-import org.apache.tez.dag.api.impl.VertexStatus.State;
-
-public class NullVertexOutputCommitter extends VertexOutputCommitter {
-
-  @Override
-  public void init(VertexContext context) throws IOException {
-    // Nothing to do
-  }
-
-  @Override
-  public void setupVertex() throws IOException {
-    // Nothing to do
-  }
-
-  @Override
-  public void commitVertex() throws IOException {
-    // Nothing to do
-  }
-
-  @Override
-  public void abortVertex(State finalState) throws IOException {
-    // Nothing to do
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
deleted file mode 100644
index 181aea5..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.impl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezVertexID;
-
-public interface VertexContext {
-
-  public Configuration getConf();
-
-  public TezDAGID getDAGId();
-  
-  // TODO Get rid of this as part of VertexContext cleanup.
-  public ApplicationAttemptId getApplicationAttemptId();
-
-  public TezVertexID getVertexId();
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
deleted file mode 100644
index fe3e2d5..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.impl;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-@Public
-@Unstable
-public abstract class VertexOutputCommitter {
-
-  /**
-   * Setup up the vertex output committer.
-   *
-   * @param context Context of the vertex whose output is being written.
-   * @throws IOException
-   */
-  public abstract void init(VertexContext context) throws IOException;
-
-  /**
-   * For the framework to setup the vertex output during initialization. This is
-   * called from the application master process for the vertex. This will be
-   * called multiple times, once per dag attempt for each vertex.
-   *
-   * @throws IOException if setup fails
-   */
-  public abstract void setupVertex() throws IOException;
-
-  /**
-   * For committing vertex's output after successful vertex completion.
-   * Note that this is invoked for vertices with a successful final state.
-   * This is called from the application master process for the entire vertex.
-   * This is guaranteed to only be called once.
-   * If it throws an exception the entire vertex will fail.
-   *
-   * @throws IOException
-   */
-  public abstract void commitVertex() throws IOException;
-
-  /**
-   * For aborting an unsuccessful vertex's output. Note that this is invoked for
-   * vertices with a final failed state. This is called from the application
-   * master process for the entire vertex. This may be called multiple times.
-   *
-   * @param state final runstate of the vertex
-   * @throws IOException
-   */
-  public abstract void abortVertex(VertexStatus.State finalState)
-      throws IOException;
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
deleted file mode 100644
index 9ba2ece..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.impl;
-
-public class VertexStatus {
-
-  public static enum State {
-    RUNNING(1),
-    SUCCEEDED(2),
-    FAILED(3),
-    PREP(4),
-    KILLED(5);
-
-    int value;
-
-    State(int value) {
-      this.value = value;
-    }
-
-    public int getValue() {
-      return value;
-    }
-
-  };
-
-  private State runState;
-
-  public synchronized State getState() {
-    return runState;
-  }
-
-  public synchronized void setState(State state) {
-    this.runState = state;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
new file mode 100644
index 0000000..7726d7b
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+public interface AMInfo {
+  public ApplicationAttemptId getAppAttemptId();
+  public long getStartTime();
+  public ContainerId getContainerId();
+  public String getNodeManagerHost();
+  public int getNodeManagerPort();
+  public int getNodeManagerHttpPort();
+
+  public void setAppAttemptId(ApplicationAttemptId appAttemptId);
+  public void setStartTime(long startTime);
+  public void setContainerId(ContainerId containerId);
+  public void setNodeManagerHost(String nmHost);
+  public void setNodeManagerPort(int nmPort);
+  public void setNodeManagerHttpPort(int mnHttpPort);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java
new file mode 100644
index 0000000..a2ee7e3
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+
+public interface TaskAttemptReport {
+  public abstract TezTaskAttemptID getTaskAttemptId();
+  public abstract TaskAttemptState getTaskAttemptState();
+  public abstract float getProgress();
+  public abstract long getStartTime();
+  public abstract long getFinishTime();
+  /** @return the shuffle finish time. Applicable only for reduce attempts */
+  public abstract long getShuffleFinishTime();
+  /** @return the sort/merge finish time. Applicable only for reduce attempts */
+  public abstract long getSortFinishTime();
+  public abstract TezCounters getCounters();
+  public abstract String getDiagnosticInfo();
+  public abstract String getStateString();
+  public abstract String getNodeManagerHost();
+  public abstract int getNodeManagerPort();
+  public abstract int getNodeManagerHttpPort();
+  public abstract ContainerId getContainerId();
+
+  public abstract void setTaskAttemptId(TezTaskAttemptID taskAttemptId);
+  public abstract void setTaskAttemptState(TaskAttemptState taskAttemptState);
+  public abstract void setProgress(float progress);
+  public abstract void setStartTime(long startTime);
+  public abstract void setFinishTime(long finishTime);
+  public abstract void setCounters(TezCounters counters);
+  public abstract void setDiagnosticInfo(String diagnosticInfo);
+  public abstract void setStateString(String stateString);
+  public abstract void setNodeManagerHost(String nmHost);
+  public abstract void setNodeManagerPort(int nmPort);
+  public abstract void setNodeManagerHttpPort(int nmHttpPort);
+  public abstract void setContainerId(ContainerId containerId);
+  
+  /** 
+   * Set the shuffle finish time. Applicable only for reduce attempts
+   * @param time the time the shuffle finished.
+   */
+  public abstract void setShuffleFinishTime(long time);
+  /** 
+   * Set the sort/merge finish time. Applicable only for reduce attempts
+   * @param time the time the shuffle finished.
+   */
+  public abstract void setSortFinishTime(long time);
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java
new file mode 100644
index 0000000..068913b
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+public enum TaskAttemptState {
+  NEW, 
+  STARTING, 
+  RUNNING, 
+  COMMIT_PENDING,  
+  SUCCEEDED,
+  FAILED,
+  KILLED
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java
new file mode 100644
index 0000000..3a0685f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+import java.util.List;
+
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+
+public interface TaskReport {
+  public abstract TezTaskID getTaskId();
+  public abstract TaskState getTaskState();
+  public abstract float getProgress();
+  public abstract long getStartTime();
+  public abstract long getFinishTime();
+  public abstract TezCounters getCounters();
+  
+  public abstract List<TezTaskAttemptID> getRunningAttemptsList();
+  public abstract TezTaskAttemptID getRunningAttempt(int index);
+  public abstract int getRunningAttemptsCount();
+  
+  public abstract TezTaskAttemptID getSuccessfulAttempt();
+  
+  public abstract List<String> getDiagnosticsList();
+  public abstract String getDiagnostics(int index);
+  public abstract int getDiagnosticsCount();
+  
+  
+  public abstract void setTaskId(TezTaskID taskId);
+  public abstract void setTaskState(TaskState taskState);
+  public abstract void setProgress(float progress);
+  public abstract void setStartTime(long startTime);
+  public abstract void setFinishTime(long finishTime);
+  public abstract void setCounters(TezCounters counters);
+  
+  public abstract void addAllRunningAttempts(List<TezTaskAttemptID> taskAttempts);
+  public abstract void addRunningAttempt(TezTaskAttemptID taskAttempt);
+  public abstract void removeRunningAttempt(int index);
+  public abstract void clearRunningAttempts();
+  
+  public abstract void setSuccessfulAttempt(TezTaskAttemptID taskAttempt);
+  public abstract void addAllDiagnostics(List<String> diagnostics);
+  public abstract void addDiagnostics(String diagnostics);
+  public abstract void removeDiagnostics(int index);
+  public abstract void clearDiagnostics();
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java
new file mode 100644
index 0000000..c6b8767
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+public enum TaskState {
+  NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILLED
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java
deleted file mode 100644
index d57754a..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.records;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-
-public interface AMInfo {
-  public ApplicationAttemptId getAppAttemptId();
-  public long getStartTime();
-  public ContainerId getContainerId();
-  public String getNodeManagerHost();
-  public int getNodeManagerPort();
-  public int getNodeManagerHttpPort();
-
-  public void setAppAttemptId(ApplicationAttemptId appAttemptId);
-  public void setStartTime(long startTime);
-  public void setContainerId(ContainerId containerId);
-  public void setNodeManagerHost(String nmHost);
-  public void setNodeManagerPort(int nmPort);
-  public void setNodeManagerHttpPort(int mnHttpPort);
-}
\ No newline at end of file


[34/50] [abbrv] git commit: TEZ-113. Add unit tests for AMContainer. (sseth)

Posted by ss...@apache.org.
TEZ-113. Add unit tests for AMContainer. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a74b4367
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a74b4367
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a74b4367

Branch: refs/heads/master
Commit: a74b43673dddf2f9d488f70b3bc337577b1c8dfa
Parents: 2340884
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed May 29 14:18:00 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed May 29 14:18:00 2013 -0700

----------------------------------------------------------------------
 .../java/org/apache/tez/dag/app/AppContext.java    |    2 +-
 .../tez/dag/app/TaskAttemptListenerImpTezDag.java  |    4 +-
 .../dag/app/rm/container/AMContainerHelpers.java   |    1 -
 .../tez/dag/app/rm/container/AMContainerImpl.java  |   19 +-
 .../tez/dag/app/rm/container/TestAMContainer.java  | 1008 +++++++++++++++
 5 files changed, 1024 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a74b4367/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
index 041de56..03e1059 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
@@ -53,7 +53,7 @@ public interface AppContext {
 
   long getStartTime();
 
-  CharSequence getUser();
+  String getUser();
 
   DAG getDAG();
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a74b4367/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index d974cd8..7600db4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -504,9 +504,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
     return COMPLETION_RESPONSE_NO_WAIT;
   }
   
-  
-  
-  // TODO EVENTUALLY remove all mrv2 ids.
+
   @Override
   public void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
     attemptToContainerIdMap.remove(attemptId);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a74b4367/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 6a9ae2e..357eeda 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -172,7 +172,6 @@ public class AMContainerHelpers {
     Map<String, String> myEnv = new HashMap<String, String>(env.size());
     myEnv.putAll(env);
     myEnv.putAll(vertexEnv);
-    // TODO TEZ-38 MRChildJVM2.setEnv should become a no-op
 
     // Set up the launch command
     List<String> commands = TezEngineChildJVM.getVMCommand(

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a74b4367/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 1089013..51246dd 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -54,8 +54,6 @@ import org.apache.tez.dag.app.rm.AMSchedulerEventContainerCompleted;
 import org.apache.tez.dag.app.rm.AMSchedulerEventDeallocateContainer;
 import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
 import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
-import org.apache.tez.dag.history.DAGHistoryEvent;
-import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 
 @SuppressWarnings("rawtypes")
@@ -156,7 +154,7 @@ public class AMContainerImpl implements AMContainer {
         .addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_TIMED_OUT, new TimedOutAtRunningTransition())
         .addTransition(AMContainerState.RUNNING, AMContainerState.STOPPING, AMContainerEventType.C_NODE_FAILED, new NodeFailedAtRunningTransition())
         .addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, AMContainerEventType.C_LAUNCHED, AMContainerEventType.C_LAUNCH_FAILED, AMContainerEventType.C_NM_STOP_SENT, AMContainerEventType.C_NM_STOP_FAILED), new ErrorAtRunningTransition())
-        
+
         .addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_ASSIGN_TA, new AssignTAAtWindDownTransition())
         .addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_PULL_TA, new PullTAAfterStopTransition())
         .addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.COMPLETED, AMContainerEventType.C_COMPLETED, new CompletedAtWindDownTransition())
@@ -168,11 +166,12 @@ public class AMContainerImpl implements AMContainer {
         
         .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_ASSIGN_TA, new AssignTAAtWindDownTransition())
         .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_PULL_TA, new PullTAAfterStopTransition())
+        // TODO This transition is wrong. Should be a noop / error.
         .addTransition(AMContainerState.STOPPING, AMContainerState.COMPLETED, AMContainerEventType.C_COMPLETED, new CompletedAtWindDownTransition())
         .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_NODE_FAILED, new NodeFailedBaseTransition())
         .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, EnumSet.of(AMContainerEventType.C_LAUNCHED, AMContainerEventType.C_LAUNCH_FAILED, AMContainerEventType.C_TA_SUCCEEDED, AMContainerEventType.C_STOP_REQUEST, AMContainerEventType.C_NM_STOP_SENT, AMContainerEventType.C_NM_STOP_FAILED, AMContainerEventType.C_TIMED_OUT))
         .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_LAUNCH_REQUEST, new ErrorAtStoppingTransition())
-        
+
         .addTransition(AMContainerState.COMPLETED, AMContainerState.COMPLETED, AMContainerEventType.C_ASSIGN_TA, new AssignTAAtCompletedTransition())
         .addTransition(AMContainerState.COMPLETED, AMContainerState.COMPLETED, AMContainerEventType.C_PULL_TA, new PullTAAfterStopTransition())
         .addTransition(AMContainerState.COMPLETED, AMContainerState.COMPLETED, AMContainerEventType.C_NODE_FAILED, new NodeFailedBaseTransition())
@@ -236,7 +235,11 @@ public class AMContainerImpl implements AMContainer {
   public List<TezTaskAttemptID> getQueuedTaskAttempts() {
     readLock.lock();
     try {
-      return Collections.singletonList(this.pendingAttempt);
+      if (pendingAttempt != null) {
+        return Collections.singletonList(this.pendingAttempt);
+      } else {
+        return Collections.emptyList();
+      }
     } finally {
       readLock.unlock();
     }
@@ -261,6 +264,10 @@ public class AMContainerImpl implements AMContainer {
       readLock.unlock();
     }
   }
+  
+  public boolean isInErrorState() {
+    return inError;
+  }
 
   @Override
   public void handle(AMContainerEvent event) {
@@ -333,6 +340,8 @@ public class AMContainerImpl implements AMContainer {
           container.taskAttemptListener, event.getCredentials(),
           event.shouldProfile(), container.appContext);
 
+      // Registering now, so that in case of delayed NM response, the child 
+      // task is not told to die since the TAL does not know about the container.
       container.registerWithTAListener();
       container.sendStartRequestToNM();
       LOG.info("Sending Launch Request for Container with id: " +

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a74b4367/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
new file mode 100644
index 0000000..574a2c5
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -0,0 +1,1008 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm.container;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.net.InetSocketAddress;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.tez.common.TezTaskContext;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.ContainerHeartbeatHandler;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventNodeFailed;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
+import org.apache.tez.dag.app.rm.AMSchedulerEventType;
+import org.apache.tez.dag.app.rm.NMCommunicatorEventType;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.engine.common.security.JobTokenIdentifier;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+public class TestAMContainer {
+
+  
+  @Test
+  // Assign before launch.
+  public void tetSingleSuccessfulTaskFlow() {
+    WrappedContainer wc = new WrappedContainer();
+    
+    wc.verifyState(AMContainerState.ALLOCATED);
+
+    // Launch request.
+    wc.launchContainer();
+    wc.verifyState(AMContainerState.LAUNCHING);
+    // 1 Launch request.
+    wc.verifyCountAndGetOutgoingEvents(1);
+    
+    // Assign task.
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.verifyState(AMContainerState.LAUNCHING);
+    wc.verifyNoOutgoingEvents();
+    assertEquals(wc.taskAttemptID, wc.amContainer.getQueuedTaskAttempts()
+        .get(0));
+    
+    // Container Launched
+    wc.containerLaunched();
+    wc.verifyState(AMContainerState.IDLE);
+    wc.verifyNoOutgoingEvents();
+    assertEquals(wc.taskAttemptID, wc.amContainer.getQueuedTaskAttempts()
+        .get(0));
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+    verify(wc.tal).registerRunningContainer(wc.containerID);
+    verify(wc.chh).register(wc.containerID);
+    
+    // Pull TA
+    AMContainerTask pulledTask = wc.pullTaskToRun();
+    wc.verifyState(AMContainerState.RUNNING);
+    wc.verifyNoOutgoingEvents();
+    assertFalse(pulledTask.shouldDie());
+    assertEquals(wc.tezTaskContext.getTaskAttemptId(), pulledTask.getTask()
+        .getTaskAttemptId());
+    assertEquals(wc.taskAttemptID, wc.amContainer.getRunningTaskAttempt());
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+    
+    wc.taskAttemptSucceeded(wc.taskAttemptID);
+    wc.verifyState(AMContainerState.IDLE);
+    wc.verifyNoOutgoingEvents();
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+    verify(wc.tal).unregisterTaskAttempt(wc.taskAttemptID);
+
+    wc.containerCompleted();
+    wc.verifyState(AMContainerState.COMPLETED);
+    // 1 Scheduler completed event.
+    wc.verifyCountAndGetOutgoingEvents(1);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+    verify(wc.chh).unregister(wc.containerID);
+
+    assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+    assertFalse(wc.amContainer.isInErrorState());
+  }
+  
+  @Test
+  // Assign after launch.
+  public void testSingleSuccessfulTaskFlow2() {
+    WrappedContainer wc = new WrappedContainer();
+    
+    wc.verifyState(AMContainerState.ALLOCATED);
+
+    // Launch request.
+    wc.launchContainer();
+    wc.verifyState(AMContainerState.LAUNCHING);
+    // 1 Launch request.
+    wc.verifyCountAndGetOutgoingEvents(1);
+    
+    // Container Launched
+    wc.containerLaunched();
+    wc.verifyState(AMContainerState.IDLE);
+    wc.verifyNoOutgoingEvents();
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+    verify(wc.tal).registerRunningContainer(wc.containerID);
+    verify(wc.chh).register(wc.containerID);
+    
+    // Assign task.
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.verifyState(AMContainerState.IDLE);
+    wc.verifyNoOutgoingEvents();
+    assertEquals(wc.taskAttemptID, wc.amContainer.getQueuedTaskAttempts()
+        .get(0));
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+
+    // Pull TA
+    AMContainerTask pulledTask = wc.pullTaskToRun();
+    wc.verifyState(AMContainerState.RUNNING);
+    wc.verifyNoOutgoingEvents();
+    assertFalse(pulledTask.shouldDie());
+    assertEquals(wc.tezTaskContext.getTaskAttemptId(), pulledTask.getTask()
+        .getTaskAttemptId());
+    assertEquals(wc.taskAttemptID, wc.amContainer.getRunningTaskAttempt());
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+
+    wc.taskAttemptSucceeded(wc.taskAttemptID);
+    wc.verifyState(AMContainerState.IDLE);
+    wc.verifyNoOutgoingEvents();
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+    verify(wc.tal).unregisterTaskAttempt(wc.taskAttemptID);
+
+    wc.containerCompleted();
+    wc.verifyState(AMContainerState.COMPLETED);
+    // 1 Scheduler completed event.
+    wc.verifyCountAndGetOutgoingEvents(1);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+    verify(wc.chh).unregister(wc.containerID);
+    
+    assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+    assertFalse(wc.amContainer.isInErrorState());
+  }
+  
+  @Test
+  public void testSingleSuccessfulTaskFlowStopRequest() {
+    WrappedContainer wc = new WrappedContainer();
+
+    wc.verifyState(AMContainerState.ALLOCATED);
+
+    wc.launchContainer();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.containerLaunched();
+    wc.pullTaskToRun();
+    wc.taskAttemptSucceeded(wc.taskAttemptID);
+
+    wc.stopRequest();
+    wc.verifyState(AMContainerState.STOP_REQUESTED);
+    // Event to NM to stop the container.
+    wc.verifyCountAndGetOutgoingEvents(1);
+    assertTrue(wc.verifyCountAndGetOutgoingEvents(1).get(0).getType() ==
+        NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+    
+    wc.nmStopSent();
+    wc.verifyState(AMContainerState.STOPPING);
+    wc.verifyNoOutgoingEvents();
+
+    wc.containerCompleted();
+    wc.verifyState(AMContainerState.COMPLETED);
+    // 1 Scheduler completed event.
+    wc.verifyCountAndGetOutgoingEvents(1);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+    verify(wc.chh).unregister(wc.containerID);
+
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+    assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+    assertFalse(wc.amContainer.isInErrorState());
+  }
+  
+  @Test
+  public void testSingleSuccessfulTaskFlowFailedNMStopRequest() {
+    WrappedContainer wc = new WrappedContainer();
+
+    wc.verifyState(AMContainerState.ALLOCATED);
+
+    wc.launchContainer();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.containerLaunched();
+    wc.pullTaskToRun();
+    wc.taskAttemptSucceeded(wc.taskAttemptID);
+
+    wc.stopRequest();
+    wc.verifyState(AMContainerState.STOP_REQUESTED);
+    // Event to NM to stop the container.
+    wc.verifyCountAndGetOutgoingEvents(1);
+    assertTrue(wc.verifyCountAndGetOutgoingEvents(1).get(0).getType() ==
+        NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+    
+    wc.nmStopFailed();
+    wc.verifyState(AMContainerState.STOPPING);
+    // Event to ask a RM container release.
+    wc.verifyCountAndGetOutgoingEvents(1);
+    assertTrue(wc.verifyCountAndGetOutgoingEvents(1).get(0).getType() ==
+        AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
+    
+    wc.containerCompleted();
+    wc.verifyState(AMContainerState.COMPLETED);
+    // 1 Scheduler completed event.
+    wc.verifyCountAndGetOutgoingEvents(1);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+    verify(wc.chh).unregister(wc.containerID);
+
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+    assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+    assertFalse(wc.amContainer.isInErrorState());
+  }
+  
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testMultipleAllocationsAtIdle() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+    
+    wc.launchContainer();
+    wc.containerLaunched();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.verifyState(AMContainerState.IDLE);
+    
+    TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+    wc.assignTaskAttempt(taID2);
+    
+    wc.verifyState(AMContainerState.STOP_REQUESTED);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+    verify(wc.chh).unregister(wc.containerID);
+    // 1 for NM stop request. 2 TERMINATING to TaskAttempt.
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATING);
+    assertTrue(wc.amContainer.isInErrorState());
+    
+    wc.nmStopSent();
+    wc.containerCompleted();
+    // 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED);
+    
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+//    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
+//    assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+  }
+  
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testAllocationAtRunning() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+    
+    wc.launchContainer();
+    wc.containerLaunched();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.pullTaskToRun();
+    wc.verifyState(AMContainerState.RUNNING);
+    
+    TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+    wc.assignTaskAttempt(taID2);
+    
+    wc.verifyState(AMContainerState.STOP_REQUESTED);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+    verify(wc.chh).unregister(wc.containerID);
+    // 1 for NM stop request. 2 TERMINATING to TaskAttempt.
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATING);
+    assertTrue(wc.amContainer.isInErrorState());
+    
+    wc.nmStopSent();
+    wc.containerCompleted();
+    // 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED);
+    
+//  assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
+//  assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
+//  assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+  }
+  
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testMultipleAllocationsAtLaunching() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+    
+    wc.launchContainer();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.pullTaskToRun();
+    wc.verifyState(AMContainerState.LAUNCHING);
+    
+    TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+    wc.assignTaskAttempt(taID2);
+    
+    wc.verifyState(AMContainerState.STOP_REQUESTED);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+    verify(wc.chh).unregister(wc.containerID);
+    // 1 for NM stop request. 2 TERMINATING to TaskAttempt.
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATING);
+    assertTrue(wc.amContainer.isInErrorState());
+    
+    wc.nmStopSent();
+    wc.containerCompleted();
+    // 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED);
+    
+//  assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
+//  assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
+//  assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+  }
+
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testContainerTimedOutAtRunning() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+    
+    wc.launchContainer();
+    wc.containerLaunched();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.pullTaskToRun();
+    wc.verifyState(AMContainerState.RUNNING);
+    
+    wc.containerTimedOut();
+    wc.verifyState(AMContainerState.STOP_REQUESTED);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+    verify(wc.chh).unregister(wc.containerID);
+    // 1 to TA, 1 for RM de-allocate.
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+        NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+    // TODO Should this be an RM DE-ALLOCATE instead ?
+    
+    wc.containerCompleted();
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED);
+    
+    assertFalse(wc.amContainer.isInErrorState());
+    
+//  assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
+//  assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
+//  assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+  }
+
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testLaunchFailure() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+
+    wc.launchContainer();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.verifyState(AMContainerState.LAUNCHING);
+    wc.launchFailed();
+    wc.verifyState(AMContainerState.STOPPING);
+    verify(wc.tal).registerRunningContainer(wc.containerID);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+        AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
+
+    wc.containerCompleted();
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+    // Valid transition. Container complete, but not with an error.
+    assertFalse(wc.amContainer.isInErrorState());
+  }
+
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testContainerCompletedAtAllocated() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+    wc.verifyState(AMContainerState.ALLOCATED);
+
+    wc.containerCompleted();
+    wc.verifyState(AMContainerState.COMPLETED);
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+    assertFalse(wc.amContainer.isInErrorState());
+  }
+
+  @Ignore
+  @SuppressWarnings("rawtypes")
+  @Test
+  // Verify that incoming NM launched events to COMPLETED containers are
+  // handled.
+  public void testContainerCompletedAtLaunching() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+
+    wc.launchContainer();
+    
+    
+    wc.assignTaskAttempt(wc.taskAttemptID);
+
+    wc.containerCompleted();
+    wc.verifyState(AMContainerState.COMPLETED);
+    verify(wc.tal).registerRunningContainer(wc.containerID);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED);
+    // TODO Failing because of an extra diagnostic event.
+    
+    assertFalse(wc.amContainer.isInErrorState());
+    
+    // Container launched generated by NM call.
+    wc.containerLaunched();
+    wc.verifyNoOutgoingEvents();
+    
+    assertFalse(wc.amContainer.isInErrorState());
+  }
+
+  @Ignore
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testContainerCompletedAtIdle() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+
+    wc.launchContainer();
+    
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.containerLaunched();
+    wc.verifyState(AMContainerState.IDLE);
+
+    wc.containerCompleted();
+    wc.verifyState(AMContainerState.COMPLETED);
+    verify(wc.tal).registerRunningContainer(wc.containerID);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+    verify(wc.chh).register(wc.containerID);
+    verify(wc.chh).unregister(wc.containerID);
+
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED);
+    // TODO Failing because of two extra diagnostic event.
+    
+    assertFalse(wc.amContainer.isInErrorState());
+    
+    // Pending pull request. (Ideally, container should be dead at this point
+    // and this event should not be generated. Network timeout on NM-RM heartbeat
+    // can cause it to be genreated)
+    wc.pullTaskToRun();
+    wc.verifyNoOutgoingEvents();
+    
+    assertFalse(wc.amContainer.isInErrorState());
+  }
+  
+  @Ignore
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testContainerCompletedAtRunning() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+
+    wc.launchContainer();
+    
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.containerLaunched();
+    wc.pullTaskToRun();
+    wc.verifyState(AMContainerState.RUNNING);
+
+    wc.containerCompleted();
+    wc.verifyState(AMContainerState.COMPLETED);
+    verify(wc.tal).registerRunningContainer(wc.containerID);
+    verify(wc.tal).unregisterRunningContainer(wc.containerID);
+    verify(wc.chh).register(wc.containerID);
+    verify(wc.chh).unregister(wc.containerID);
+
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED);
+    // TODO Failing because of two extra diagnostic event.
+    
+    assertFalse(wc.amContainer.isInErrorState());
+    
+    // Pending task complete. (Ideally, container should be dead at this point
+    // and this event should not be generated. Network timeout on NM-RM heartbeat
+    // can cause it to be genreated)
+    wc.taskAttemptSucceeded(wc.taskAttemptID);
+    wc.verifyNoOutgoingEvents();
+    
+    assertFalse(wc.amContainer.isInErrorState());
+  }
+
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testTaskAssignedToCompletedContainer() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+    
+    wc.launchContainer();
+    wc.containerLaunched();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.pullTaskToRun();
+    wc.taskAttemptSucceeded(wc.taskAttemptID);
+    
+    wc.containerCompleted();
+    wc.verifyState(AMContainerState.COMPLETED);
+    
+    TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+    
+    wc.assignTaskAttempt(taID2);
+    
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED);
+    TaskAttemptEventContainerTerminated ctEvent = 
+        (TaskAttemptEventContainerTerminated) outgoingEvents.get(0);
+    assertEquals(taID2, ctEvent.getTaskAttemptID());
+  
+    // Allocation to a completed Container is considered an error.
+    // TODO Is this valid ?
+    assertTrue(wc.amContainer.isInErrorState());
+  }
+
+  @Test
+  public void testTaskPullAtLaunching() {
+    WrappedContainer wc = new WrappedContainer();
+    
+    wc.launchContainer();
+    AMContainerTask pulledTask = wc.pullTaskToRun();
+    wc.verifyState(AMContainerState.LAUNCHING);
+    wc.verifyNoOutgoingEvents();
+    assertFalse(pulledTask.shouldDie());
+    assertNull(pulledTask.getTask());
+  }
+  
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testNodeFailedAtIdle() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+    
+    wc.launchContainer();
+    wc.containerLaunched();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.verifyState(AMContainerState.IDLE);
+    
+    wc.nodeFailed();
+    // Expecting a complete event from the RM
+    wc.verifyState(AMContainerState.STOPPING);
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_NODE_FAILED,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+        AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
+    
+    for (Event event : outgoingEvents) {
+      if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
+        TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
+        assertEquals("nodeFailed", nfEvent.getDiagnosticInfo());
+      }
+    }
+
+    wc.containerCompleted();
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+    assertFalse(wc.amContainer.isInErrorState());
+  }
+  
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testNodeFailedAtIdleMultipleAttempts() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+
+    wc.launchContainer();
+    wc.containerLaunched();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.pullTaskToRun();
+    wc.taskAttemptSucceeded(wc.taskAttemptID);
+    wc.verifyState(AMContainerState.IDLE);
+
+    TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+    wc.assignTaskAttempt(taID2);
+    wc.pullTaskToRun();
+    wc.taskAttemptSucceeded(taID2);
+    wc.verifyState(AMContainerState.IDLE);
+
+    wc.nodeFailed();
+    // Expecting a complete event from the RM
+    wc.verifyState(AMContainerState.STOPPING);
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_NODE_FAILED,
+        TaskAttemptEventType.TA_NODE_FAILED,
+        AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
+
+    for (Event event : outgoingEvents) {
+      if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
+        TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
+        assertEquals("nodeFailed", nfEvent.getDiagnosticInfo());
+      }
+    }
+
+    assertFalse(wc.amContainer.isInErrorState());
+
+    wc.containerCompleted();
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED);
+    
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+    assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size());
+  }
+  
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testNodeFailedAtRunningMultipleAttempts() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+    
+    wc.launchContainer();
+    wc.containerLaunched();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.pullTaskToRun();
+    wc.taskAttemptSucceeded(wc.taskAttemptID);
+    
+    TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+    wc.assignTaskAttempt(taID2);
+    wc.pullTaskToRun();
+    wc.verifyState(AMContainerState.RUNNING);
+    
+    wc.nodeFailed();
+    // Expecting a complete event from the RM
+    wc.verifyState(AMContainerState.STOPPING);
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(4);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_NODE_FAILED,
+        TaskAttemptEventType.TA_NODE_FAILED,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+        AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
+    
+    for (Event event : outgoingEvents) {
+      if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
+        TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
+        assertEquals("nodeFailed", nfEvent.getDiagnosticInfo());
+      }
+    }
+    
+    wc.containerCompleted();
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+    assertFalse(wc.amContainer.isInErrorState());
+//    assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO. Set/Unset properly.
+//    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
+//    assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+  }
+  
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testNodeFailedAtCompletedMultipleSuccessfulTAs() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+    
+    wc.launchContainer();
+    wc.containerLaunched();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.pullTaskToRun();
+    wc.taskAttemptSucceeded(wc.taskAttemptID);
+    
+    TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+    wc.assignTaskAttempt(taID2);
+    wc.pullTaskToRun();
+    wc.taskAttemptSucceeded(taID2);
+    wc.stopRequest();
+    wc.nmStopSent();
+    wc.containerCompleted();
+    wc.verifyState(AMContainerState.COMPLETED);
+    
+    wc.nodeFailed();
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        TaskAttemptEventType.TA_NODE_FAILED,
+        TaskAttemptEventType.TA_NODE_FAILED);
+    
+    assertNull(wc.amContainer.getRunningTaskAttempt());
+    assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+    assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size());
+  }
+
+  @SuppressWarnings("rawtypes")
+  @Test
+  public void testDuplicateCompletedEvents() {
+    WrappedContainer wc = new WrappedContainer();
+    List<Event> outgoingEvents;
+    
+    wc.launchContainer();
+    wc.containerLaunched();
+    wc.assignTaskAttempt(wc.taskAttemptID);
+    wc.pullTaskToRun();
+    wc.taskAttemptSucceeded(wc.taskAttemptID);
+    
+    TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+    wc.assignTaskAttempt(taID2);
+    wc.pullTaskToRun();
+    wc.taskAttemptSucceeded(taID2);
+    wc.stopRequest();
+    wc.nmStopSent();
+    wc.containerCompleted();
+    wc.verifyState(AMContainerState.COMPLETED);
+    
+    outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
+    verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+        AMSchedulerEventType.S_CONTAINER_COMPLETED);
+   
+    wc.containerCompleted();
+    wc.verifyNoOutgoingEvents();
+  }
+  
+  
+  // TODO Verify diagnostics in most of the tests.
+  
+  private static class WrappedContainer {
+    
+    long rmIdentifier = 2000;
+    ApplicationId applicationID;
+    ApplicationAttemptId appAttemptID;
+    ContainerId containerID;
+    NodeId nodeID;
+    String nodeHttpAddress;
+    Resource resource;
+    Priority priority;
+    Container container;
+    ContainerHeartbeatHandler chh;
+    TaskAttemptListener tal;
+    
+    @SuppressWarnings("rawtypes")
+    EventHandler eventHandler;
+
+    AppContext appContext;
+
+    TezDAGID dagID;
+    TezVertexID vertexID;
+    TezTaskID taskID;
+    TezTaskAttemptID taskAttemptID;
+    
+    TezTaskContext tezTaskContext;
+
+    Token<JobTokenIdentifier> jobToken;
+    
+    public AMContainerImpl amContainer;
+    
+    @SuppressWarnings("unchecked")
+    public WrappedContainer() {
+      applicationID = BuilderUtils.newApplicationId(rmIdentifier, 1);
+      appAttemptID = BuilderUtils.newApplicationAttemptId(applicationID, 1);
+      containerID = BuilderUtils.newContainerId(appAttemptID, 1);
+      nodeID = BuilderUtils.newNodeId("host", 12500);
+      nodeHttpAddress = "host:12501";
+      resource = BuilderUtils.newResource(1024, 1);
+      priority = BuilderUtils.newPriority(1);
+      container = BuilderUtils.newContainer(containerID, nodeID,
+          nodeHttpAddress, resource, priority, null, rmIdentifier);
+      chh = mock(ContainerHeartbeatHandler.class);
+      
+      InetSocketAddress addr = new InetSocketAddress("localhost", 0);
+      tal = mock(TaskAttemptListener.class);
+      doReturn(addr).when(tal).getAddress();
+      
+      eventHandler = mock(EventHandler.class);
+      
+      appContext = mock(AppContext.class);
+      doReturn(new HashMap<ApplicationAccessType, String>()).when(appContext)
+      .getApplicationACLs();
+      doReturn(eventHandler).when(appContext).getEventHandler();
+
+      dagID = new TezDAGID(applicationID, 1);
+      vertexID = new TezVertexID(dagID, 1);
+      taskID = new TezTaskID(vertexID, 1);
+      taskAttemptID = new TezTaskAttemptID(taskID, 1);
+      
+      tezTaskContext = mock(TezTaskContext.class);
+      doReturn(taskAttemptID).when(tezTaskContext).getTaskAttemptId();
+
+      
+      jobToken = (Token<JobTokenIdentifier>) mock(Token.class);
+      
+      amContainer = new AMContainerImpl(container, chh, tal,
+          appContext);
+    }
+
+    /**
+     * Verifies no additional outgoing events generated by the last incoming
+     * event to the AMContainer.
+     */
+    @SuppressWarnings("unchecked")
+    public void verifyNoOutgoingEvents() {
+      verify(eventHandler, never()).handle(any(Event.class));
+    }
+    
+    /**
+     * Returns a list of outgoing events generated by the last incoming event to
+     * the AMContainer. 
+     * @param invocations number of expected invocations.
+     * 
+     * @return a list of outgoing events from the AMContainer.
+     */
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    public List<Event> verifyCountAndGetOutgoingEvents(int invocations) {
+      ArgumentCaptor<Event> args = ArgumentCaptor.forClass(Event.class);
+      verify(eventHandler, times(invocations)).handle(args.capture());
+      return args.getAllValues();
+    }
+
+    public void launchContainer() {
+      reset(eventHandler);
+      amContainer.handle(new AMContainerEventLaunchRequest(containerID, vertexID,
+          jobToken, new Credentials(), false, new TezConfiguration(),
+          new HashMap<String, LocalResource>(), new HashMap<String, String>(),
+          null));
+    }
+    
+    public void assignTaskAttempt(TezTaskAttemptID taID) {
+      reset(eventHandler);
+      amContainer.handle(new AMContainerEventAssignTA(containerID, taID,
+          tezTaskContext)); 
+    }
+    
+    public AMContainerTask pullTaskToRun() {
+      reset(eventHandler);
+      return amContainer.pullTaskContext();
+    }
+
+    public void containerLaunched() {
+      reset(eventHandler);
+      amContainer.handle(new AMContainerEventLaunched(containerID, 3000));
+    }
+
+    public void taskAttemptSucceeded(TezTaskAttemptID taID) {
+      reset(eventHandler);
+      amContainer.handle(new AMContainerEventTASucceeded(containerID, taID));
+    }
+    
+    public void stopRequest() {
+      reset(eventHandler);
+      amContainer.handle(new AMContainerEvent(containerID,
+          AMContainerEventType.C_STOP_REQUEST));
+    }
+    
+    public void nmStopSent() {
+      reset(eventHandler);
+      amContainer.handle(new AMContainerEvent(containerID,
+          AMContainerEventType.C_NM_STOP_SENT));
+    }
+    
+    public void nmStopFailed() {
+      reset(eventHandler);
+      amContainer.handle(new AMContainerEvent(containerID,
+          AMContainerEventType.C_NM_STOP_FAILED));
+    }
+    
+    public void containerCompleted() {
+      reset(eventHandler);
+      ContainerStatus cStatus = ContainerStatus.newInstance(containerID,
+          ContainerState.COMPLETE, "", 100);
+      amContainer.handle(new AMContainerEventCompleted(cStatus));
+    }
+    
+    public void containerTimedOut() {
+      reset(eventHandler);
+      amContainer.handle(new AMContainerEvent(containerID,
+          AMContainerEventType.C_TIMED_OUT));
+    }
+    
+    public void launchFailed() {
+      reset(eventHandler);
+      amContainer.handle(new AMContainerEventLaunchFailed(containerID,
+          "launchFailed"));
+    }
+    
+    public void nodeFailed() {
+      reset(eventHandler);
+      amContainer.handle(new AMContainerEventNodeFailed(containerID,
+          "nodeFailed"));
+    }
+    
+    public void verifyState(AMContainerState state) {
+      assertEquals(
+          "Expected state: " + state + ", but found: " + amContainer.getState(),
+          state, amContainer.getState());
+    }
+  }
+
+  @SuppressWarnings("rawtypes")
+  private void verifyUnOrderedOutgoingEventTypes(List<Event> events,
+      Enum<?>... expectedTypes) {
+
+    List<Enum<?>> expectedTypeList = new LinkedList<Enum<?>>();
+    for (Enum<?> expectedType : expectedTypes) {
+      expectedTypeList.add(expectedType);
+    }
+    List<Event> eventsCopy = new LinkedList<Event>(events);
+
+    Iterator<Enum<?>> expectedTypeIterator = expectedTypeList.iterator();
+    while (expectedTypeIterator.hasNext()) {
+      Enum<?> expectedType = expectedTypeIterator.next();
+      Iterator<Event> iter = eventsCopy.iterator();
+      while (iter.hasNext()) {
+        Event e = iter.next();
+        if (e.getType() == expectedType) {
+          iter.remove();
+          expectedTypeIterator.remove();
+          break;
+        }
+      }
+    }
+    assertTrue("Did not find types : " + expectedTypeList
+        + " in outgoing event list", expectedTypeList.isEmpty());
+    assertTrue("Found unexpected events: " + eventsCopy
+        + " in outgoing event list", eventsCopy.isEmpty());
+  }
+}


[14/50] [abbrv] TEZ-9. Add support for client api and records. cleaned up packages (bikas)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
deleted file mode 100644
index a5a9e73..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.records;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-
-public interface TaskAttemptReport {
-  public abstract TezTaskAttemptID getTaskAttemptId();
-  public abstract TaskAttemptState getTaskAttemptState();
-  public abstract float getProgress();
-  public abstract long getStartTime();
-  public abstract long getFinishTime();
-  /** @return the shuffle finish time. Applicable only for reduce attempts */
-  public abstract long getShuffleFinishTime();
-  /** @return the sort/merge finish time. Applicable only for reduce attempts */
-  public abstract long getSortFinishTime();
-  public abstract TezCounters getCounters();
-  public abstract String getDiagnosticInfo();
-  public abstract String getStateString();
-  public abstract String getNodeManagerHost();
-  public abstract int getNodeManagerPort();
-  public abstract int getNodeManagerHttpPort();
-  public abstract ContainerId getContainerId();
-
-  public abstract void setTaskAttemptId(TezTaskAttemptID taskAttemptId);
-  public abstract void setTaskAttemptState(TaskAttemptState taskAttemptState);
-  public abstract void setProgress(float progress);
-  public abstract void setStartTime(long startTime);
-  public abstract void setFinishTime(long finishTime);
-  public abstract void setCounters(TezCounters counters);
-  public abstract void setDiagnosticInfo(String diagnosticInfo);
-  public abstract void setStateString(String stateString);
-  public abstract void setNodeManagerHost(String nmHost);
-  public abstract void setNodeManagerPort(int nmPort);
-  public abstract void setNodeManagerHttpPort(int nmHttpPort);
-  public abstract void setContainerId(ContainerId containerId);
-  
-  /** 
-   * Set the shuffle finish time. Applicable only for reduce attempts
-   * @param time the time the shuffle finished.
-   */
-  public abstract void setShuffleFinishTime(long time);
-  /** 
-   * Set the sort/merge finish time. Applicable only for reduce attempts
-   * @param time the time the shuffle finished.
-   */
-  public abstract void setSortFinishTime(long time);
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptState.java
deleted file mode 100644
index 455aa1b..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptState.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.records;
-
-public enum TaskAttemptState {
-  NEW, 
-  STARTING, 
-  RUNNING, 
-  COMMIT_PENDING,  
-  SUCCEEDED,
-  FAILED,
-  KILLED
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
deleted file mode 100644
index a0834d7..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.records;
-
-import java.util.List;
-
-import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-
-public interface TaskReport {
-  public abstract TezTaskID getTaskId();
-  public abstract TaskState getTaskState();
-  public abstract float getProgress();
-  public abstract long getStartTime();
-  public abstract long getFinishTime();
-  public abstract TezCounters getCounters();
-  
-  public abstract List<TezTaskAttemptID> getRunningAttemptsList();
-  public abstract TezTaskAttemptID getRunningAttempt(int index);
-  public abstract int getRunningAttemptsCount();
-  
-  public abstract TezTaskAttemptID getSuccessfulAttempt();
-  
-  public abstract List<String> getDiagnosticsList();
-  public abstract String getDiagnostics(int index);
-  public abstract int getDiagnosticsCount();
-  
-  
-  public abstract void setTaskId(TezTaskID taskId);
-  public abstract void setTaskState(TaskState taskState);
-  public abstract void setProgress(float progress);
-  public abstract void setStartTime(long startTime);
-  public abstract void setFinishTime(long finishTime);
-  public abstract void setCounters(TezCounters counters);
-  
-  public abstract void addAllRunningAttempts(List<TezTaskAttemptID> taskAttempts);
-  public abstract void addRunningAttempt(TezTaskAttemptID taskAttempt);
-  public abstract void removeRunningAttempt(int index);
-  public abstract void clearRunningAttempts();
-  
-  public abstract void setSuccessfulAttempt(TezTaskAttemptID taskAttempt);
-  public abstract void addAllDiagnostics(List<String> diagnostics);
-  public abstract void addDiagnostics(String diagnostics);
-  public abstract void removeDiagnostics(int index);
-  public abstract void clearDiagnostics();
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskState.java
deleted file mode 100644
index 411a8f4..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskState.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.records;
-
-public enum TaskState {
-  NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILLED
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index a0c2676..804c20c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
@@ -63,10 +64,14 @@ import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.service.CompositeService;
 import org.apache.hadoop.yarn.service.Service;
 import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.VertexStatus;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.app.client.ClientService;
 import org.apache.tez.dag.app.client.impl.TezClientService;
 import org.apache.tez.dag.app.dag.DAG;
@@ -173,6 +178,7 @@ public class DAGAppMaster extends CompositeService {
   private HistoryEventHandler historyEventHandler;
 
   private DAGAppMasterState state;
+  private DAGMonitorServer monitor;
 
   private DAG dag;
   private Credentials fsTokens = new Credentials(); // Filled during init
@@ -223,6 +229,8 @@ public class DAGAppMaster extends CompositeService {
     appName = jobPlan.getName();
 
     dagId = new TezDAGID(appAttemptID.getApplicationId(), 1);
+    
+    monitor = new DAGMonitorServer();
 
     // TODO Committer.
     //    committer = createOutputCommitter(conf);
@@ -703,6 +711,35 @@ public class DAGAppMaster extends CompositeService {
     }
     LOG.info("On DAG completion. Old state: " + oldState + " new state: " + state);
   }
+  
+  class DAGMonitorServer implements DAGClient {
+
+    @Override
+    public List<String> getAllDAGs() {
+      return Collections.singletonList(dag.getID().toString());
+    }
+
+    @Override
+    public DAGStatus getDAGStatus(String dagIdStr) throws TezException {
+      return getDAG(dagIdStr).getDAGStatus();
+    }
+
+    @Override
+    public VertexStatus getVertexStatus(String dagIdStr, String vertexName) {
+      return getDAG(dagIdStr).getVertexStatus(vertexName);
+    }
+    
+    DAG getDAG(String dagIdStr) {
+      TezDAGID dagId = TezDAGID.fromString(dagIdStr);
+      if(dagId == null) {
+        throw new TezException("Bad dagId: " + dagIdStr);
+      }
+      if(!dagId.equals(dag.getID())) {
+        throw new TezException("Unknown dagId: " + dagIdStr);
+      }
+      return dag;
+    }
+  }
 
   private class RunningAppContext implements AppContext {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
index 0d845b2..92f0e87 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
@@ -17,7 +17,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.Edge;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index f2c3791..913b565 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -24,9 +24,10 @@ import java.util.Map;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.client.DAGStatusBuilder;
+import org.apache.tez.dag.api.client.VertexStatusBuilder;
 import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezVertexID;
 
@@ -61,6 +62,8 @@ public interface DAG {
   TezConfiguration getConf();
   
   DAGPlan getJobPlan();
+  DAGStatusBuilder getDAGStatus();
+  VertexStatusBuilder getVertexStatus(String vertexName);
   
   /**
    * @return the ACLs for this job for each type of JobACL given. 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java
index 1fe18fa..49ab24a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.app.dag;
 import java.util.List;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.tez.dag.api.records.AMInfo;
+import org.apache.tez.dag.api.oldrecords.AMInfo;
 
 public abstract class DAGReport {
   public abstract ApplicationId getDAGId();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
index 6cfc2c2..f3ca831 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
@@ -21,8 +21,8 @@ package org.apache.tez.dag.app.dag;
 import java.util.Map;
 
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.records.TaskReport;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.oldrecords.TaskReport;
+import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
index d4e038a..712f968 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
@@ -25,8 +25,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.records.TaskAttemptReport;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptReport;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index ace8a8b..fc86753 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -28,7 +28,9 @@ import org.apache.tez.common.InputSpec;
 import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.EdgeProperty;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.client.ProgressBuilder;
+import org.apache.tez.dag.api.client.VertexStatusBuilder;
 import org.apache.tez.dag.records.TezTaskID;
 import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
@@ -59,6 +61,8 @@ public interface Vertex extends Comparable<Vertex> {
   int getTotalTasks();
   int getCompletedTasks();
   float getProgress();
+  ProgressBuilder getVertexProgress();
+  VertexStatusBuilder getVertexStatus();
   
   TezDependentTaskCompletionEvent[]
       getTaskAttemptCompletionEvents(int fromEventId, int maxEvents);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
index ef23638..2659bba 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.app.dag.event;
 import java.util.List;
 
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskAttemptEventStatusUpdate extends TaskAttemptEvent {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
index 79512c7..4bf4e24 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.app.dag.event;
 
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.records.TezTaskID;
 
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 02e91b4..66f9c36 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -52,14 +52,16 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGProtos.EdgePlan;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint;
-import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
-import org.apache.tez.dag.api.impl.DAGStatus;
+import org.apache.tez.dag.api.client.DAGStatusBuilder;
+import org.apache.tez.dag.api.client.ProgressBuilder;
+import org.apache.tez.dag.api.client.VertexStatusBuilder;
+import org.apache.tez.dag.api.committer.DAGStatus;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
@@ -84,6 +86,7 @@ import org.apache.tez.dag.history.events.DAGStartedEvent;
 import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.dag.utils.DAGApps;
+import org.apache.tez.dag.utils.TezBuilderUtils;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
 import org.apache.tez.engine.common.security.JobTokenSecretManager;
 import org.apache.tez.engine.common.security.TokenCache;
@@ -505,7 +508,52 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
       readLock.unlock();
     }
   }
+  
+  // monitoring apis
+  @Override
+  public DAGStatusBuilder getDAGStatus() {
+    DAGStatusBuilder status = new DAGStatusBuilder();
+    int totalTaskCount = 0;
+    int totalSucceededTaskCount = 0;
+    int totalRunningTaskCount = 0;
+    int totalFailedTaskCount = 0;
+    int totalKilledTaskCount = 0;
+    readLock.lock();
+    try {
+      for(Map.Entry<String, Vertex> entry : vertexMap.entrySet()) {
+        ProgressBuilder progress = entry.getValue().getVertexProgress();
+        status.addVertexProgress(entry.getKey(), progress);
+        totalTaskCount += progress.getTotalTaskCount();
+        totalSucceededTaskCount += progress.getSucceededTaskCount();
+        totalRunningTaskCount += progress.getRunningTaskCount();
+        totalFailedTaskCount += progress.getFailedTaskCount();
+        totalKilledTaskCount += progress.getKilledTaskCount();
+      }
+      ProgressBuilder dagProgress = new ProgressBuilder();
+      dagProgress.setTotalTaskCount(totalTaskCount);
+      dagProgress.setSucceededTaskCount(totalSucceededTaskCount);
+      dagProgress.setRunningTaskCount(totalRunningTaskCount);
+      dagProgress.setFailedTaskCount(totalFailedTaskCount);
+      dagProgress.setKilledTaskCount(totalKilledTaskCount);
+      status.setState(getState());
+      status.setDiagnostics(diagnostics);
+      status.setDAGProgress(dagProgress);
+      return status;
+    } finally {
+      readLock.unlock();
+    }
+  }
 
+  @Override
+  public VertexStatusBuilder getVertexStatus(String vertexName) {
+    Vertex vertex = vertexMap.get(vertexName);
+    if(vertex == null) {
+      return null;
+    }
+    return vertex.getVertexStatus();
+  }
+  
+  
   protected void startRootVertices() {
     for (Vertex v : vertices.values()) {
       if (v.getInputVerticesCount() == 0) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 264b3ad..92ccffe 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -56,9 +56,8 @@ import org.apache.tez.common.counters.DAGCounter;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
-import org.apache.tez.dag.api.records.TaskAttemptReport;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptReport;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
@@ -93,6 +92,7 @@ import org.apache.tez.dag.history.events.TaskAttemptFinishedEvent;
 import org.apache.tez.dag.history.events.TaskAttemptStartedEvent;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.utils.TezBuilderUtils;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
 
 import com.google.common.annotations.VisibleForTesting;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 4d0c4f6..401f1b0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -43,9 +43,9 @@ import org.apache.hadoop.yarn.util.Records;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.records.TaskAttemptState;
-import org.apache.tez.dag.api.records.TaskReport;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskReport;
+import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index aedb386..20d13cd 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -55,19 +55,21 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.tez.common.InputSpec;
 import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.oldrecords.TaskState;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.impl.NullVertexOutputCommitter;
-import org.apache.tez.dag.api.impl.VertexContext;
-import org.apache.tez.dag.api.impl.VertexOutputCommitter;
-import org.apache.tez.dag.api.impl.VertexStatus;
-import org.apache.tez.dag.api.impl.VertexStatus.State;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.client.ProgressBuilder;
+import org.apache.tez.dag.api.client.VertexStatusBuilder;
+import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
+import org.apache.tez.dag.api.committer.VertexContext;
+import org.apache.tez.dag.api.committer.VertexOutputCommitter;
+import org.apache.tez.dag.api.committer.VertexStatus;
+import org.apache.tez.dag.api.committer.VertexStatus.State;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
@@ -519,6 +521,36 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       this.readLock.unlock();
     }
   }
+  
+  @Override
+  public ProgressBuilder getVertexProgress() {
+    this.readLock.lock();
+    try {
+      ProgressBuilder progress = new ProgressBuilder();
+      progress.setTotalTaskCount(numTasks);
+      progress.setSucceededTaskCount(succeededTaskCount);
+      progress.setRunningTaskCount(0); // TODO TEZ-130
+      progress.setFailedTaskCount(failedTaskCount);
+      progress.setKilledTaskCount(killedTaskCount);
+      return progress;
+    } finally {
+      this.readLock.unlock();
+    }
+  }
+  
+  @Override
+  public VertexStatusBuilder getVertexStatus() {
+    this.readLock.lock();
+    try {
+      VertexStatusBuilder status = new VertexStatusBuilder();
+      status.setState(getInternalState());
+      status.setDiagnostics(diagnostics);
+      status.setProgress(getVertexProgress());
+      return status;
+    } finally {
+      this.readLock.unlock();
+    }
+  }
 
   private void computeProgress() {
     this.readLock.lock();
@@ -599,9 +631,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       writeLock.unlock();
     }
   }
-
-  @Private
-  public VertexState getInternalState() {
+  
+  private VertexState getInternalState() {
     readLock.lock();
     try {
      return getStateMachine().getCurrentState();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
index fddd204..b2a90f2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
@@ -18,7 +18,7 @@
 package org.apache.tez.dag.app.rm;
 
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.app.dag.TaskAttempt;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 31005b6..85ba0b2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.service.AbstractService;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.DAGAppMaster;
 import org.apache.tez.dag.app.DAGAppMasterState;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
index 04c104a..79ccb8e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
@@ -38,8 +38,7 @@ import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.service.AbstractService;
-import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.Task;
@@ -50,6 +49,7 @@ import org.apache.tez.dag.app.dag.event.TaskEventType;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
 import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.dag.utils.TezBuilderUtils;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 // FIXME does not handle multiple vertices

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
index aa95f37..34bb4e3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
@@ -23,7 +23,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.Task;
 import org.apache.tez.dag.app.dag.TaskAttempt;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
index 191d0b9..ba5bfda 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
@@ -26,7 +26,7 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.Task;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
index d1b87b1..5dea9a7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.history.events;
 
-import org.apache.tez.dag.api.impl.DAGStatus;
+import org.apache.tez.dag.api.committer.DAGStatus;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.DAGFinished;
 import org.apache.tez.dag.history.avro.HistoryEventType;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
index 4a2917e..395ba93 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.history.events;
 
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
 import org.apache.tez.dag.history.avro.TaskAttemptFinished;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
index 165dc20..60f810b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.history.events;
 
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
 import org.apache.tez.dag.history.avro.TaskFinished;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
index b02be1e..91c209f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.history.events;
 
-import org.apache.tez.dag.api.impl.VertexStatus;
+import org.apache.tez.dag.api.committer.VertexStatus;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
 import org.apache.tez.dag.history.avro.VertexFinished;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
new file mode 100644
index 0000000..60ec0a0
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.utils;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.dag.api.oldrecords.AMInfo;
+import org.apache.tez.dag.app.dag.DAGReport;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+
+public class TezBuilderUtils {
+
+  public static TezVertexID newVertexID(TezDAGID dagId, int vertexId) {
+    return new TezVertexID(dagId, vertexId);
+  }
+
+  public static TezTaskAttemptID newTaskAttemptId(TezTaskID taskId, int id) {
+    return new TezTaskAttemptID(taskId, id);
+  }
+  
+  public static DAGReport newDAGReport() {
+    return null;
+  }
+
+  public static AMInfo newAMInfo(ApplicationAttemptId appAttemptID, 
+      long startTime, ContainerId containerID, String nmHost, 
+      int nmPort, int nmHttpPort) {
+    return null;
+  }
+
+  public static TezTaskID newTaskId(TezDAGID dagId, int vertexId, int taskId) {
+    return new TezTaskID(newVertexID(dagId, vertexId), taskId);
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index 369bced..9601418 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -56,7 +56,7 @@ import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.common.TezTaskContext;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerHeartbeatHandler;
 import org.apache.tez.dag.app.TaskAttemptListener;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 100f2aa..44e7686 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -43,8 +43,8 @@ import org.apache.hadoop.yarn.event.InlineDispatcher;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.records.TaskAttemptState;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index a1e450c..fe6ae14 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -37,23 +37,23 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.event.DrainDispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.EdgePlan;
-import org.apache.tez.dag.api.DAGProtos.PlanEdgeConnectionPattern;
-import org.apache.tez.dag.api.DAGProtos.PlanEdgeSourceType;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
-import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.oldrecords.TaskState;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeConnectionPattern;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeSourceType;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint;
-import org.apache.tez.dag.api.impl.NullVertexOutputCommitter;
-import org.apache.tez.dag.api.impl.VertexContext;
-import org.apache.tez.dag.api.impl.VertexOutputCommitter;
-import org.apache.tez.dag.api.impl.VertexStatus.State;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
+import org.apache.tez.dag.api.committer.VertexContext;
+import org.apache.tez.dag.api.committer.VertexOutputCommitter;
+import org.apache.tez.dag.api.committer.VertexStatus.State;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index c0c9923..fcb2ca1 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -99,7 +99,7 @@ import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.Edge;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;


[37/50] [abbrv] git commit: Include container log urls and counters as part of history events in the AM logs.

Posted by ss...@apache.org.
Include container log urls and counters as part of history events in the AM logs.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/6d5625f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/6d5625f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/6d5625f4

Branch: refs/heads/master
Commit: 6d5625f4fa38cf35e6f7201fb4b18c47ec6964d3
Parents: 34de34b
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu May 30 13:34:38 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu May 30 13:34:38 2013 -0700

----------------------------------------------------------------------
 tez-dag/src/main/avro/HistoryEvents.avpr           |   35 +++++++-
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |    5 +-
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java      |   27 ++++++-
 .../org/apache/tez/dag/app/dag/impl/TaskImpl.java  |    5 +-
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |    6 +-
 .../apache/tez/dag/history/events/AvroUtils.java   |   53 ++++++++++++
 .../tez/dag/history/events/DAGFinishedEvent.java   |    9 ++-
 .../history/events/TaskAttemptFinishedEvent.java   |   10 ++-
 .../history/events/TaskAttemptStartedEvent.java    |   11 ++-
 .../tez/dag/history/events/TaskFinishedEvent.java  |    9 ++-
 .../dag/history/events/VertexFinishedEvent.java    |   10 ++-
 .../tez/dag/app/rm/container/TestAMContainer.java  |    3 +-
 .../apache/tez/mapreduce/examples/MRRSleepJob.java |   65 +++++++++------
 13 files changed, 200 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/avro/HistoryEvents.avpr
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/avro/HistoryEvents.avpr b/tez-dag/src/main/avro/HistoryEvents.avpr
index 04333e6..8805c6d 100644
--- a/tez-dag/src/main/avro/HistoryEvents.avpr
+++ b/tez-dag/src/main/avro/HistoryEvents.avpr
@@ -21,6 +21,29 @@
 
  "types": [
 
+     {"type": "record", "name": "TezCounter",
+      "fields": [
+          {"name": "name", "type": "string"},
+          {"name": "displayName", "type": "string"},
+          {"name": "value", "type": "long"}
+      ]
+     },
+
+     {"type": "record", "name": "TezCounterGroup",
+      "fields": [
+          {"name": "name", "type": "string"},
+          {"name": "displayName", "type": "string"},
+          {"name": "counts", "type": {"type": "array", "items": "TezCounter"}}
+      ]
+     },
+
+     {"type": "record", "name": "TezCounters",
+      "fields": [
+          {"name": "groups", "type": {"type": "array", "items": "TezCounterGroup"}}
+      ]
+     },
+
+
      {"type": "record", "name": "AMStarted",
       "fields": [
           {"name": "applicationAttemptId", "type": "string"},
@@ -50,7 +73,8 @@
           {"name": "dagId", "type": "string"},
           {"name": "finishTime", "type": "long"},
           {"name": "status", "type": "string"},
-          {"name": "diagnostics", "type": "string"}
+          {"name": "diagnostics", "type": "string"},
+          {"name": "counters", "type": "TezCounters"}
       ]
      },
 
@@ -71,7 +95,8 @@
           {"name": "vertexId", "type": "string"},
           {"name": "finishTime", "type": "long"},
           {"name": "status", "type": "string"},
-          {"name": "diagnostics", "type": "string"}
+          {"name": "diagnostics", "type": "string"},
+          {"name": "counters", "type": "TezCounters"}
       ]
      },
 
@@ -89,7 +114,8 @@
           {"name": "vertexName", "type": "string"},
           {"name": "taskId", "type": "string"},
           {"name": "finishTime", "type": "long"},
-          {"name": "status", "type": "string"}
+          {"name": "status", "type": "string"},
+          {"name": "counters", "type": "TezCounters"}
       ]
      },
 
@@ -109,7 +135,8 @@
           {"name": "taskAttemptId", "type": "string"},
           {"name": "finishTime", "type": "long"},
           {"name": "status", "type": "string"},
-          {"name": "diagnostics", "type": "string"}
+          {"name": "diagnostics", "type": "string"},
+          {"name": "counters", "type": "TezCounters"}
       ]
      },
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 3681c85..d2395ad 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -625,7 +625,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   void logJobHistoryFinishedEvent() {
     this.setFinishTime();
     DAGFinishedEvent finishEvt = new DAGFinishedEvent(dagId, finishTime,
-        DAGStatus.State.SUCCEEDED, "");
+        DAGStatus.State.SUCCEEDED, "", getAllCounters());
     this.eventHandler.handle(
         new DAGHistoryEvent(dagId, finishEvt));
   }
@@ -641,7 +641,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
 
   void logJobHistoryUnsuccesfulEvent(DAGStatus.State state) {
     DAGFinishedEvent finishEvt = new DAGFinishedEvent(dagId, clock.getTime(),
-        state, StringUtils.join(LINE_SEPARATOR, getDiagnostics()));
+        state, StringUtils.join(LINE_SEPARATOR, getDiagnostics()),
+        getAllCounters());
     this.eventHandler.handle(
         new DAGHistoryEvent(dagId, finishEvt));
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 5225ecf..c14ba8a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
@@ -794,9 +795,27 @@ public class TaskAttemptImpl implements TaskAttempt,
 
   @SuppressWarnings("unchecked")
   protected void logJobHistoryAttemptStarted() {
+    final String containerIdStr = containerId.toString();
+    String inProgressLogsUrl = nodeHttpAddress
+       + "/" + "node/containerlogs"
+       + "/" + containerIdStr
+       + "/" + this.appContext.getUser();
+    String completedLogsUrl = "";
+    if (conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
+        YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)
+        && conf.get(YarnConfiguration.YARN_LOG_SERVER_URL) != null) {
+      String contextStr = "v_" + getTask().getVertex().getName()
+          + "_" + this.attemptId.toString();
+      completedLogsUrl = conf.get(YarnConfiguration.YARN_LOG_SERVER_URL)
+          + "/" + containerNodeId.toString()
+          + "/" + containerIdStr
+          + "/" + contextStr
+          + "/" + this.appContext.getUser();
+    }
     TaskAttemptStartedEvent startEvt = new TaskAttemptStartedEvent(
         attemptId, getTask().getVertex().getName(),
-        launchTime, containerId, containerNodeId);
+        launchTime, containerId, containerNodeId,
+        inProgressLogsUrl, completedLogsUrl);
     eventHandler.handle(new DAGHistoryEvent(
         attemptId.getTaskID().getVertexID().getDAGId(),
         startEvt));
@@ -809,7 +828,8 @@ public class TaskAttemptImpl implements TaskAttempt,
     
     TaskAttemptFinishedEvent finishEvt = new TaskAttemptFinishedEvent(
         attemptId, getTask().getVertex().getName(),
-        getFinishTime(), TaskAttemptState.SUCCEEDED, "");
+        getFinishTime(), TaskAttemptState.SUCCEEDED, "",
+        getCounters());
     // FIXME how do we store information regd completion events
     eventHandler.handle(new DAGHistoryEvent(
         attemptId.getTaskID().getVertexID().getDAGId(),
@@ -823,7 +843,8 @@ public class TaskAttemptImpl implements TaskAttempt,
         attemptId, getTask().getVertex().getName(),
         clock.getTime(), state,
         StringUtils.join(
-            LINE_SEPARATOR, getDiagnostics()));
+            LINE_SEPARATOR, getDiagnostics()),
+        getCounters());
     // FIXME how do we store information regd completion events
     eventHandler.handle(new DAGHistoryEvent(
         attemptId.getTaskID().getVertexID().getDAGId(),

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index fb738ce..575f32c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -806,14 +806,15 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     // FIXME need to handle getting finish time as this function
     // is called from within a transition
     TaskFinishedEvent finishEvt = new TaskFinishedEvent(taskId,
-        getVertex().getName(), clock.getTime(), TaskState.SUCCEEDED);
+        getVertex().getName(), clock.getTime(), TaskState.SUCCEEDED,
+        getCounters());
     this.eventHandler.handle(new DAGHistoryEvent(
         taskId.getVertexID().getDAGId(), finishEvt));
   }
   
   protected void logJobHistoryTaskFailedEvent(TaskState finalState) {
     TaskFinishedEvent finishEvt = new TaskFinishedEvent(taskId,
-        getVertex().getName(), clock.getTime(), finalState);
+        getVertex().getName(), clock.getTime(), finalState, getCounters());
     this.eventHandler.handle(new DAGHistoryEvent(
         taskId.getVertexID().getDAGId(), finishEvt));
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index e34382f..c7bceb6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -670,14 +670,16 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   void logJobHistoryVertexFinishedEvent() {
     this.setFinishTime();
     VertexFinishedEvent finishEvt = new VertexFinishedEvent(vertexId,
-        vertexName, finishTime, VertexStatus.State.SUCCEEDED, "");
+        vertexName, finishTime, VertexStatus.State.SUCCEEDED, "",
+        getAllCounters());
     this.eventHandler.handle(new DAGHistoryEvent(getDAGId(), finishEvt));
   }
 
   void logJobHistoryVertexFailedEvent(VertexStatus.State state) {
     VertexFinishedEvent finishEvt = new VertexFinishedEvent(vertexId,
         vertexName, clock.getTime(), state,
-        StringUtils.join(LINE_SEPARATOR, getDiagnostics()));
+        StringUtils.join(LINE_SEPARATOR, getDiagnostics()),
+        getAllCounters());
     this.eventHandler.handle(new DAGHistoryEvent(getDAGId(), finishEvt));
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/AvroUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AvroUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AvroUtils.java
new file mode 100644
index 0000000..3587964
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AvroUtils.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.history.events;
+
+import java.util.ArrayList;
+
+import org.apache.avro.util.Utf8;
+import org.apache.tez.common.counters.CounterGroup;
+import org.apache.tez.dag.history.avro.TezCounter;
+import org.apache.tez.dag.history.avro.TezCounterGroup;
+import org.apache.tez.dag.history.avro.TezCounters;
+
+public class AvroUtils {
+
+  public static TezCounters toAvro(
+      org.apache.tez.common.counters.TezCounters counters) {
+    TezCounters result = new TezCounters();
+    result.groups = new ArrayList<TezCounterGroup>(0);
+    if (counters == null) return result;
+    for (CounterGroup group : counters) {
+      TezCounterGroup g = new TezCounterGroup();
+      g.name = new Utf8(group.getName());
+      g.displayName = new Utf8(group.getDisplayName());
+      g.counts = new ArrayList<TezCounter>(group.size());
+      for (org.apache.tez.common.counters.TezCounter counter : group) {
+        TezCounter c = new TezCounter();
+        c.name = new Utf8(counter.getName());
+        c.displayName = new Utf8(counter.getDisplayName());
+        c.value = counter.getValue();
+        g.counts.add(c);
+      }
+      result.groups.add(g);
+    }
+    return result;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
index 8794cea..8ded3d4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
@@ -18,6 +18,7 @@
 
 package org.apache.tez.dag.history.events;
 
+import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.DAGFinished;
@@ -27,14 +28,17 @@ import org.apache.tez.dag.records.TezDAGID;
 public class DAGFinishedEvent implements HistoryEvent {
 
   private DAGFinished datum = new DAGFinished();
+  // FIXME remove this when we have a proper history
+  private final TezCounters tezCounters;
 
   public DAGFinishedEvent(TezDAGID dagId,
       long finishTime, DAGStatus.State state,
-      String diagnostics) {
+      String diagnostics, TezCounters counters) {
     datum.dagId = dagId.toString();
     datum.finishTime = finishTime;
     datum.status = state.name();
     datum.diagnostics = diagnostics;
+    tezCounters = counters;
   }
 
   @Override
@@ -58,6 +62,7 @@ public class DAGFinishedEvent implements HistoryEvent {
     return "dagId=" + datum.dagId
         + ", finishTime=" + datum.finishTime
         + ", status=" + datum.status
-        + ", diagnostics=" + datum.diagnostics;
+        + ", diagnostics=" + datum.diagnostics
+        + ", counters=" + tezCounters.toString();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
index 395ba93..9794938 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
@@ -18,6 +18,7 @@
 
 package org.apache.tez.dag.history.events;
 
+import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
@@ -27,17 +28,21 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
 public class TaskAttemptFinishedEvent implements HistoryEvent {
 
   private TaskAttemptFinished datum = new TaskAttemptFinished();
+  // FIXME remove this when we have a proper history
+  private final TezCounters tezCounters;
 
   public TaskAttemptFinishedEvent(TezTaskAttemptID taId,
       String vertexName,
       long finishTime,
       TaskAttemptState state,
-      String diagnostics) {
+      String diagnostics,
+      TezCounters counters) {
     datum.taskAttemptId = taId.toString();
     datum.vertexName = vertexName;
     datum.finishTime = finishTime;
     datum.status = state.name();
     datum.diagnostics = diagnostics;
+    tezCounters = counters;
   }
 
   @Override
@@ -62,6 +67,7 @@ public class TaskAttemptFinishedEvent implements HistoryEvent {
         + ", taskAttemptId=" + datum.taskAttemptId
         + ", finishTime=" + datum.finishTime
         + ", status=" + datum.status
-        + ", diagnostics=" + datum.diagnostics;
+        + ", diagnostics=" + datum.diagnostics
+        + ", counters=" + tezCounters.toString();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
index 97c9f2f..5d29b21 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
@@ -28,15 +28,20 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
 public class TaskAttemptStartedEvent implements HistoryEvent {
 
   private TaskAttemptStarted datum = new TaskAttemptStarted();
+  private final String inProgressLogsUrl;
+  private final String completedLogsUrl;
 
   public TaskAttemptStartedEvent(TezTaskAttemptID taId,
       String vertexName, long startTime,
-      ContainerId containerId, NodeId nodeId) {
+      ContainerId containerId, NodeId nodeId,
+      String inProgressLogsUrl, String completedLogsUrl) {
     datum.taskAttemptId = taId.toString();
     datum.vertexName = vertexName;
     datum.startTime = startTime;
     datum.containerId = containerId.toString();
     datum.nodeId = nodeId.toString();
+    this.inProgressLogsUrl = inProgressLogsUrl;
+    this.completedLogsUrl = completedLogsUrl;
   }
 
   @Override
@@ -61,6 +66,8 @@ public class TaskAttemptStartedEvent implements HistoryEvent {
         + ", taskAttemptId=" + datum.taskAttemptId
         + ", startTime=" + datum.startTime
         + ", containerId=" + datum.containerId
-        + ", nodeId=" + datum.nodeId;
+        + ", nodeId=" + datum.nodeId
+        + ", inProgressLogs=" + inProgressLogsUrl
+        + ", completedLogs=" + completedLogsUrl;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
index 60f810b..5408a72 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
@@ -18,6 +18,7 @@
 
 package org.apache.tez.dag.history.events;
 
+import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
@@ -27,14 +28,17 @@ import org.apache.tez.dag.records.TezTaskID;
 public class TaskFinishedEvent implements HistoryEvent {
 
   private TaskFinished datum = new TaskFinished();
+  // FIXME remove this when we have a proper history
+  private final TezCounters tezCounters;
 
   public TaskFinishedEvent(TezTaskID taskId,
       String vertexName, long finishTime,
-      TaskState state) {
+      TaskState state, TezCounters counters) {
     datum.vertexName = vertexName;
     datum.taskId = taskId.toString();
     datum.finishTime = finishTime;
     datum.status = state.name();
+    tezCounters = counters;
   }
 
   @Override
@@ -58,6 +62,7 @@ public class TaskFinishedEvent implements HistoryEvent {
     return "vertexName=" + datum.vertexName
         + ", taskId=" + datum.taskId
         + ", finishTime=" + datum.finishTime
-        + ", status=" + datum.status;
+        + ", status=" + datum.status
+        + ", counters=" + tezCounters.toString();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
index 7e86c66..ec13ebd 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
@@ -18,6 +18,7 @@
 
 package org.apache.tez.dag.history.events;
 
+import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.client.VertexStatus;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
@@ -27,15 +28,19 @@ import org.apache.tez.dag.records.TezVertexID;
 public class VertexFinishedEvent implements HistoryEvent {
 
   private VertexFinished datum = new VertexFinished();
+  // FIXME remove this when we have a proper history
+  private final TezCounters tezCounters;
 
   public VertexFinishedEvent(TezVertexID vertexId,
       String vertexName, long finishTime,
-      VertexStatus.State state, String diagnostics) {
+      VertexStatus.State state, String diagnostics,
+      TezCounters counters) {
     datum.vertexName = vertexName;
     datum.vertexId = vertexId.toString();
     datum.finishTime = finishTime;
     datum.status = state.name();
     datum.diagnostics = diagnostics;
+    tezCounters = counters;
   }
 
   @Override
@@ -60,6 +65,7 @@ public class VertexFinishedEvent implements HistoryEvent {
         + ", vertexId=" + datum.vertexId
         + ", finishTime=" + datum.finishTime
         + ", status=" + datum.status
-        + ", diagnostics=" + datum.diagnostics;
+        + ", diagnostics=" + datum.diagnostics
+        + ", counters=" + tezCounters.toString();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index 4541a2e..cf01f43 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -839,7 +839,8 @@ public class TestAMContainer {
       resource = BuilderUtils.newResource(1024, 1);
       priority = BuilderUtils.newPriority(1);
       container = BuilderUtils.newContainer(containerID, nodeID,
-          nodeHttpAddress, resource, priority, null, rmIdentifier);
+          nodeHttpAddress, resource, priority, null);
+
       chh = mock(ContainerHeartbeatHandler.class);
       
       InetSocketAddress addr = new InetSocketAddress("localhost", 0);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
index 6115c6f..627d583 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -72,6 +72,8 @@ public class MRRSleepJob extends Configured implements Tool {
       "mrr.sleepjob.ireduce.sleep.time";
   public static String IREDUCE_STAGES_COUNT =
       "mrr.sleepjob.ireduces.stages.count";
+  public static String IREDUCE_TASKS_COUNT =
+      "mrr.sleepjob.ireduces.tasks.count";
 
   // Flags to inject failures
   public static String MAP_THROW_ERROR = "mrr.sleepjob.map.throw.error";
@@ -109,28 +111,21 @@ public class MRRSleepJob extends Configured implements Tool {
         InputSplit ignored, TaskAttemptContext taskContext)
         throws IOException {
       Configuration conf = taskContext.getConfiguration();
-      String vertexName = conf.get(
-          org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
-      boolean isIntermediateReduce =
-          MultiStageMRConfigUtil.getIntermediateStageNum(vertexName) != -1;
-
-      final int count =
-          (isIntermediateReduce)?
-              conf.getInt(IREDUCE_SLEEP_COUNT, 1) :
-              conf.getInt(MAP_SLEEP_COUNT, 1);
-      if (count < 0) throw new IOException("Invalid map count: " + count);
 
-      int totalIReduces = MultiStageMRConfigUtil.getNumIntermediateStages(conf);
-      boolean finalIReduce = totalIReduces ==
-          (MultiStageMRConfigUtil.getIntermediateStageNum(vertexName) + 1);
+      final int count = conf.getInt(MAP_SLEEP_COUNT, 1);
+      if (count < 0) {
+        throw new IOException("Invalid map count: " + count);
+      }
 
-      final int redcount = finalIReduce?
-          conf.getInt(REDUCE_SLEEP_COUNT, 1) :
-            conf.getInt(IREDUCE_SLEEP_COUNT, 1);
-      if (redcount < 0)
-        throw new IOException("Invalid reduce count: " + redcount);
+      int totalIReduces = conf.getInt(IREDUCE_STAGES_COUNT, 1);
 
-      final int emitPerMapTask = (redcount * taskContext.getNumReduceTasks());
+      int reduceTasks = totalIReduces == 0?
+          taskContext.getNumReduceTasks() :
+            conf.getInt(IREDUCE_TASKS_COUNT, 1);
+      int sleepCount = totalIReduces == 0?
+          conf.getInt(REDUCE_SLEEP_COUNT,1) :
+            conf.getInt(IREDUCE_SLEEP_COUNT,1);
+      final int emitPerMapTask = sleepCount * reduceTasks;
 
       return new RecordReader<IntWritable,IntWritable>() {
         private int records = 0;
@@ -219,6 +214,9 @@ public class MRRSleepJob extends Configured implements Tool {
                ) throws IOException, InterruptedException {
       //it is expected that every map processes mapSleepCount number of records.
       try {
+        LOG.info("Reading in " + vertexName
+            + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+            + " key " + key.get());
         LOG.info("Sleeping in InitialMap"
             + ", vertexName=" + vertexName
             + ", taskAttemptId=" + context.getTaskAttemptID()
@@ -228,7 +226,9 @@ public class MRRSleepJob extends Configured implements Tool {
             + (mapSleepDuration * (mapSleepCount - count)));
         context.setStatus("Sleeping... (" +
           (mapSleepDuration * (mapSleepCount - count)) + ") ms left");
-        Thread.sleep(mapSleepDuration);
+        if ((mapSleepCount - count) > 0) {
+          Thread.sleep(mapSleepDuration);
+        }
         if (throwError || throwFatal) {
           throw new IOException("Throwing a simulated error from map");
         }
@@ -242,6 +242,9 @@ public class MRRSleepJob extends Configured implements Tool {
       // each reducer will get reduceSleepCount number of keys.
       int k = key.get();
       for (int i = 0; i < value.get(); ++i) {
+        LOG.info("Writing in " + vertexName
+            + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+            + " key " + (k+i) + " value 1");
         context.write(new IntWritable(k + i), new IntWritable(1));
       }
     }
@@ -269,6 +272,10 @@ public class MRRSleepJob extends Configured implements Tool {
         Context context)
             throws IOException, InterruptedException {
       try {
+        LOG.info("Reading in " + vertexName
+            + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+            + " key " + key.get());
+
         LOG.info("Sleeping in IntermediateReduce"
             + ", vertexName=" + vertexName
             + ", taskAttemptId=" + context.getTaskAttemptID()
@@ -278,7 +285,9 @@ public class MRRSleepJob extends Configured implements Tool {
             + (iReduceSleepDuration * (iReduceSleepCount - count)));
         context.setStatus("Sleeping... (" +
           (iReduceSleepDuration * (iReduceSleepCount - count)) + ") ms left");
-        Thread.sleep(iReduceSleepDuration);
+        if ((iReduceSleepCount - count) > 0) {
+          Thread.sleep(iReduceSleepDuration);
+        }
       }
       catch (InterruptedException ex) {
         throw (IOException)new IOException(
@@ -290,6 +299,9 @@ public class MRRSleepJob extends Configured implements Tool {
       int k = key.get();
       for (IntWritable value : values) {
         for (int i = 0; i < value.get(); ++i) {
+          LOG.info("Writing in " + vertexName
+              + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+              + " key " + (k+i) + " value 1");
           context.write(new IntWritable(k + i), new IntWritable(1));
         }
       }
@@ -318,6 +330,9 @@ public class MRRSleepJob extends Configured implements Tool {
                        Context context)
       throws IOException {
       try {
+        LOG.info("Reading in " + vertexName
+            + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+            + " key " + key.get());
         LOG.info("Sleeping in FinalReduce"
             + ", vertexName=" + vertexName
             + ", taskAttemptId=" + context.getTaskAttemptID()
@@ -327,8 +342,9 @@ public class MRRSleepJob extends Configured implements Tool {
             + (reduceSleepDuration * (reduceSleepCount - count)));
         context.setStatus("Sleeping... (" +
             (reduceSleepDuration * (reduceSleepCount - count)) + ") ms left");
-        Thread.sleep(reduceSleepDuration);
-
+        if ((reduceSleepCount - count) > 0) {
+          Thread.sleep(reduceSleepDuration);
+        }
       }
       catch (InterruptedException ex) {
         throw (IOException)new IOException(
@@ -357,6 +373,7 @@ public class MRRSleepJob extends Configured implements Tool {
     conf.setInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
     conf.setInt(MRJobConfig.NUM_MAPS, numMapper);
     conf.setInt(IREDUCE_STAGES_COUNT, iReduceStagesCount);
+    conf.setInt(IREDUCE_TASKS_COUNT, numIReducer);
 
     // Configure intermediate reduces
     conf.setInt(
@@ -364,7 +381,7 @@ public class MRRSleepJob extends Configured implements Tool {
         iReduceStagesCount);
     LOG.info("Running MRR with " + iReduceStagesCount + " IR stages");
 
-    for (int i = 0; i < iReduceStagesCount; ++i) {
+    for (int i = 1; i <= iReduceStagesCount; ++i) {
       // Set reducer class for intermediate reduce
       conf.setClass(
           MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,


[06/50] [abbrv] TEZ-123. Cleanup package dependencies. (bikas)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
index 12ad74c..ccfbd78 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.Path;
 import org.apache.tez.common.Constants;
 import org.apache.tez.common.TezJobConfig;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
 
 /**
  * Manipulate the working area for the transient store for maps and reduces.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutput.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutput.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutput.java
index 9f6e10b..eb1fe5f 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutput.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutput.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.fs.Path;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
 
 /**
  * Manipulate the working area for the transient store for maps and reduces.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
index eeca130..1bd65e2 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.Path;
 import org.apache.tez.common.Constants;
 import org.apache.tez.common.TezJobConfig;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
 
 /**
  * Manipulate the working area for the transient store for maps and reduces.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/tez-mapreduce/pom.xml b/tez-mapreduce/pom.xml
index 2a1ca15..e785f6b 100644
--- a/tez-mapreduce/pom.xml
+++ b/tez-mapreduce/pom.xml
@@ -27,11 +27,15 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.tez</groupId>
-      <artifactId>tez-engine</artifactId>
+      <artifactId>tez-dag-api</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.tez</groupId>
-      <artifactId>tez-dag-api</artifactId>
+      <artifactId>tez-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-engine</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 9dfaf1f..63472d2 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -71,13 +71,13 @@ import org.apache.tez.common.TezTaskStatus;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.common.records.ProceedToCompletionResponse;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.api.Task;
 import org.apache.tez.engine.common.task.local.output.TezLocalTaskOutputFiles;
 import org.apache.tez.engine.common.task.local.output.TezTaskOutput;
 import org.apache.tez.engine.lib.input.LocalMergedInput;
 import org.apache.tez.engine.lib.output.LocalOnFileSorterOutput;
 import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
 import org.apache.tez.engine.runtime.RuntimeUtils;
 import org.apache.tez.mapreduce.hadoop.IDConverter;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
index 050f608..b8a99ee 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
 import org.apache.tez.common.counters.TaskCounter;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.api.Input;
 import org.apache.tez.engine.api.Master;
 import org.apache.tez.engine.api.Output;
@@ -46,7 +47,6 @@ import org.apache.tez.engine.common.ConfigUtils;
 import org.apache.tez.engine.common.combine.CombineInput;
 import org.apache.tez.engine.common.combine.CombineOutput;
 import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 import org.apache.tez.mapreduce.hadoop.IDConverter;
 import org.apache.tez.mapreduce.hadoop.mapred.TaskAttemptContextImpl;
 import org.apache.tez.mapreduce.processor.MRTask;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
index 4c56bc4..4046596 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
@@ -24,10 +24,10 @@ import org.apache.hadoop.mapred.TaskID;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 
 public class IDConverter {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRTaskStatus.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRTaskStatus.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRTaskStatus.java
index b5c06eb..e63b704 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRTaskStatus.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRTaskStatus.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.mapred.TaskStatus;
 import org.apache.tez.common.TezTaskStatus;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class MRTaskStatus implements TezTaskStatus {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
index 0b2774d..1fc71b0 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
@@ -28,9 +28,9 @@ import org.apache.tez.common.TezTaskStatus.Phase;
 import org.apache.tez.common.counters.CounterGroup;
 import org.apache.tez.common.counters.TezCounter;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 
 public class TezTypeConverters {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java
index a3e31b0..2f06986 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.JobContext;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.util.Progressable;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.mapreduce.hadoop.IDConverter;
 
 @InterfaceAudience.Private

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java
index aacfc60..d953891 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.mapred.TaskID;
 import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.util.Progressable;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.mapreduce.processor.MRTaskReporter;
 
 @InterfaceAudience.Private

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
index d27f996..1f34fdb 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Progressable;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.mapreduce.hadoop.IDConverter;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 import org.apache.tez.mapreduce.processor.MRTaskReporter;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
index bd39b67..d7d721f 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
@@ -63,11 +63,11 @@ import org.apache.tez.common.counters.TaskCounter;
 import org.apache.tez.common.counters.TezCounter;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.common.records.ProceedToCompletionResponse;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.api.Master;
 import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator;
 import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 import org.apache.tez.mapreduce.combine.MRCombiner;
 import org.apache.tez.mapreduce.hadoop.IDConverter;
 import org.apache.tez.mapreduce.hadoop.MRConfig;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/TezTaskReporterImpl.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/TezTaskReporterImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/TezTaskReporterImpl.java
index c601109..d602459 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/TezTaskReporterImpl.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/TezTaskReporterImpl.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
 import org.apache.tez.common.counters.TezCounter;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
 
 @InterfaceAudience.Private

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
index 4796646..025bf61 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
@@ -28,8 +28,8 @@ import org.apache.tez.common.ContainerTask;
 import org.apache.tez.common.TezTaskStatus;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
 import org.apache.tez.common.records.ProceedToCompletionResponse;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
 
 public class TestUmbilicalProtocol implements TezTaskUmbilicalProtocol {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
index 4b04d13..2e823e6 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
@@ -18,10 +18,10 @@
 package org.apache.tez.mapreduce;
 
 import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 
 public class TezTestUtils {
 


[30/50] [abbrv] git commit: TEZ-142. DAG does not throw error when it receives an unknown DAGSchedulerUpdateEvent (bikas)

Posted by ss...@apache.org.
TEZ-142. DAG does not throw error when it receives an unknown DAGSchedulerUpdateEvent (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a2a72613
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a2a72613
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a2a72613

Branch: refs/heads/master
Commit: a2a726139342c5b510053a0432927a3c5f932666
Parents: 4beb98f
Author: Bikas Saha <bi...@apache.org>
Authored: Fri May 24 17:17:33 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Fri May 24 17:17:33 2013 -0700

----------------------------------------------------------------------
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |    4 +++-
 1 files changed, 3 insertions(+), 1 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a2a72613/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 5f47818..4c63d8f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -57,6 +57,7 @@ import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.client.DAGStatusBuilder;
 import org.apache.tez.dag.api.client.ProgressBuilder;
@@ -1177,7 +1178,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
         dag.dagScheduler.scheduleTask(sEvent);
         break;
       default:
-        LOG.warn("Unknown DAGEventSchedulerUpdate:" + sEvent.getUpdateType());
+        throw new TezException("Unknown DAGEventSchedulerUpdate:"
+                                + sEvent.getUpdateType());
     }
   }
 }


[17/50] [abbrv] git commit: TEZ-134. Untangle MR configuration in YarnTezDagChild. (sseth)

Posted by ss...@apache.org.
TEZ-134. Untangle MR configuration in YarnTezDagChild. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/be6d4bc0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/be6d4bc0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/be6d4bc0

Branch: refs/heads/master
Commit: be6d4bc096d9bae363cbcf1a3dc5ec67b057507d
Parents: 0d5027d
Author: Siddharth Seth <ss...@apache.org>
Authored: Sat May 18 20:51:06 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Sat May 18 20:51:06 2013 -0700

----------------------------------------------------------------------
 .../java/org/apache/tez/common/TezJobConfig.java   |   19 +-
 .../org/apache/hadoop/mapred/YarnTezDagChild.java  |  291 ++++-----------
 .../main/java/org/apache/tez/engine/api/Task.java  |    2 +
 .../apache/tez/common/TezEngineTaskContext.java    |    5 +
 .../tez/engine/common/shuffle/impl/Shuffle.java    |    2 +-
 .../tez/engine/common/sort/impl/TezMerger.java     |    2 +-
 .../task/local/output/TezLocalTaskOutputFiles.java |    4 +-
 .../task/local/output/TezTaskOutputFiles.java      |    2 +-
 .../apache/tez/engine/runtime/RuntimeUtils.java    |   22 +-
 .../org/apache/tez/engine/task/RuntimeTask.java    |   21 +-
 .../org/apache/hadoop/mapred/LocalJobRunner.java   |    8 +-
 .../apache/tez/mapreduce/task/MRRuntimeTask.java   |  262 +++++++++++++
 .../apache/tez/mapreduce/processor/MapUtils.java   |   50 +++
 .../mapreduce/processor/map/TestMapProcessor.java  |   23 +-
 .../processor/reduce/TestReduceProcessor.java      |   23 +-
 15 files changed, 477 insertions(+), 259 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
index 4a7abea..0d6f6be 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -62,11 +62,16 @@ public class TezJobConfig {
   public static final long DEFAULT_RECORDS_BEFORE_PROGRESS = 10000; 
 
   /**
-   * 
+   * List of directories avialble to the engine. 
    */
-  public static final String LOCAL_DIR = "tez.engine.local.dir";
-  public static final String DEFAULT_LOCAL_DIR = "/tmp";
-
+  public static final String LOCAL_DIRS = "tez.engine.local.dirs";
+  public static final String DEFAULT_LOCAL_DIRS = "/tmp";
+  
+  /**
+   * One local dir for the speicfic job.
+   */
+  public static final String JOB_LOCAL_DIR = "tez.engine.job.local.dir";
+  
   /**
    * The directory which contains the localized files for this task.
    */
@@ -74,6 +79,8 @@ public class TezJobConfig {
   public static final String TASK_LOCAL_RESOURCE_DIR = "tez.engine.task-local-resource.dir";
   public static final String DEFAULT_TASK_LOCAL_RESOURCE_DIR = "/tmp";
   
+  public static final String TEZ_TASK_WORKING_DIR = "tez.engine.task.working.dir";
+  
   /**
    * 
    */
@@ -299,4 +306,8 @@ public class TezJobConfig {
    * credentials.
    */
   public static final String DAG_CREDENTIALS_BINARY =  "tez.dag.credentials.binary";
+  
+  
+  public static final String APPLICATION_ATTEMPT_ID =
+      "tez.job.application.attempt.id";
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 2295b6a..6fd6eff 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -20,17 +20,9 @@ package org.apache.hadoop.mapred;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
 import java.net.InetSocketAddress;
-import java.net.URI;
 import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.crypto.SecretKey;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -39,16 +31,8 @@ import org.apache.hadoop.fs.FSError;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
-import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.source.JvmMetrics;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
@@ -72,16 +56,15 @@ import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.api.Task;
+import org.apache.tez.engine.common.security.JobTokenIdentifier;
+import org.apache.tez.engine.common.security.TokenCache;
 import org.apache.tez.engine.runtime.RuntimeUtils;
-import org.apache.tez.mapreduce.hadoop.DeprecatedKeys;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
-import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+import org.apache.tez.engine.task.RuntimeTask;
 import org.apache.tez.mapreduce.input.SimpleInput;
 import org.apache.tez.mapreduce.output.SimpleOutput;
-import org.apache.tez.mapreduce.processor.MRTask;
 
 /**
- * The main() for TEZ MapReduce task processes.
+ * The main() for TEZ Task processes.
  */
 public class YarnTezDagChild {
 
@@ -91,11 +74,9 @@ public class YarnTezDagChild {
     Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
     LOG.debug("Child starting");
 
-    DeprecatedKeys.init();
-    
-    final JobConf defaultConf = new JobConf();
-    // HACK Eventually load the DagConf for security etc setup.
-//    defaultConf.addResource(MRJobConfig.JOB_CONF_FILE);
+    final Configuration defaultConf = new Configuration();
+    // Security settings will be loaded based on core-site and core-default. Don't
+    // depend on the jobConf for this.
     UserGroupInformation.setConfiguration(defaultConf);
 
     String host = args[0];
@@ -121,9 +102,10 @@ public class YarnTezDagChild {
     // Create TaskUmbilicalProtocol as actual task owner.
     UserGroupInformation taskOwner =
       UserGroupInformation.createRemoteUser(appID.toString());
-    Token<JobTokenIdentifier> jt = TokenCache.getJobToken(credentials);
-    SecurityUtil.setTokenService(jt, address);
-    taskOwner.addToken(jt);
+
+    Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(credentials);
+    SecurityUtil.setTokenService(jobToken, address);
+    taskOwner.addToken(jobToken);
     final TezTaskUmbilicalProtocol umbilical =
       taskOwner.doAs(new PrivilegedExceptionAction<TezTaskUmbilicalProtocol>() {
       @Override
@@ -142,9 +124,8 @@ public class YarnTezDagChild {
     ContainerTask containerTask = null;
     UserGroupInformation childUGI = null;
     TezTaskAttemptID taskAttemptId = null;
-    MRTask task = null;
     ContainerContext containerContext = new ContainerContext(containerId, pid);
-    
+
     try {
       while (true) {
         // poll for new task
@@ -165,19 +146,19 @@ public class YarnTezDagChild {
         }
         taskContext = (TezEngineTaskContext) containerTask
             .getTezEngineTaskContext();
-        LOG.info("XXXX: New container task context:"
+        LOG.info("DEBUG: New container task context:"
                 + taskContext.toString());
 
         taskAttemptId = taskContext.getTaskAttemptId();
 
         final Task t = createAndConfigureTezTask(taskContext, umbilical,
-            credentials, jt,
+            credentials, jobToken,
             containerId.getApplicationAttemptId().getAttemptId());
-        task = (MRTask) t.getProcessor();
-        final JobConf job = task.getConf();
 
-        // Initiate Java VM metrics
-        JvmMetrics.initSingleton(containerId.toString(), job.getSessionId());
+        final Configuration conf = ((RuntimeTask)t).getConfiguration();
+
+        // TODO Initiate Java VM metrics
+        // JvmMetrics.initSingleton(containerId.toString(), job.getSessionId());
         childUGI = UserGroupInformation.createRemoteUser(System
             .getenv(ApplicationConstants.Environment.USER.toString()));
         // Add tokens to new user so that it may execute its task correctly.
@@ -186,7 +167,7 @@ public class YarnTezDagChild {
         childUGI.doAs(new PrivilegedExceptionAction<Object>() {
           @Override
           public Object run() throws Exception {
-            runTezTask(t, umbilical, job); // run the task
+            runTezTask(t, umbilical, conf); // run the task
             return null;
           }
         });
@@ -196,34 +177,6 @@ public class YarnTezDagChild {
     } catch (FSError e) {
       LOG.fatal("FSError from child", e);
       umbilical.fsError(taskAttemptId, e.getMessage());
-    } catch (Exception exception) {
-      LOG.warn("Exception running child : "
-          + StringUtils.stringifyException(exception));
-      try {
-        if (task != null) {
-          // do cleanup for the task
-          if (childUGI == null) { // no need to job into doAs block
-            task.taskCleanup(umbilical);
-          } else {
-            final MRTask taskFinal = task;
-            childUGI.doAs(new PrivilegedExceptionAction<Object>() {
-              @Override
-              public Object run() throws Exception {
-                taskFinal.taskCleanup(umbilical);
-                return null;
-              }
-            });
-          }
-        }
-      } catch (Exception e) {
-        LOG.info("Exception cleaning up: " + StringUtils.stringifyException(e));
-      }
-      // Report back any failures, for diagnostic purposes
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      exception.printStackTrace(new PrintStream(baos));
-      if (taskAttemptId != null) {
-        umbilical.fatalError(taskAttemptId, baos.toString());
-      }
     } catch (Throwable throwable) {
       LOG.fatal("Error running child : "
     	        + StringUtils.stringifyException(throwable));
@@ -249,29 +202,36 @@ public class YarnTezDagChild {
    * out an output directory.
    * @throws IOException 
    */
-  private static void configureLocalDirs(JobConf job) throws IOException {
+  /**
+   * Configure tez-local-dirs, tez-localized-file-dir, etc. Also create these
+   * dirs.
+   */
+  
+  private static void configureLocalDirs(Configuration conf) throws IOException {
     String[] localSysDirs = StringUtils.getTrimmedStrings(
         System.getenv(Environment.LOCAL_DIRS.name()));
-    job.setStrings(TezJobConfig.LOCAL_DIR, localSysDirs);
-    job.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
+    conf.setStrings(TezJobConfig.LOCAL_DIRS, localSysDirs);
+    conf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
         System.getenv(Environment.PWD.name()));
-    LOG.info(TezJobConfig.LOCAL_DIR + " for child: " +
-        job.get(TezJobConfig.LOCAL_DIR));
+    
+    LOG.info(TezJobConfig.LOCAL_DIRS + " for child: " +
+        conf.get(TezJobConfig.LOCAL_DIRS));
     LOG.info(TezJobConfig.TASK_LOCAL_RESOURCE_DIR + " for child: "
-        + job.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
-    LocalDirAllocator lDirAlloc = new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
+        + conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
+    
+    LocalDirAllocator lDirAlloc = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
     Path workDir = null;
     // First, try to find the JOB_LOCAL_DIR on this host.
     try {
-      workDir = lDirAlloc.getLocalPathToRead("work", job);
+      workDir = lDirAlloc.getLocalPathToRead("work", conf);
     } catch (DiskErrorException e) {
       // DiskErrorException means dir not found. If not found, it will
       // be created below.
     }
     if (workDir == null) {
       // JOB_LOCAL_DIR doesn't exist on this host -- Create it.
-      workDir = lDirAlloc.getLocalPathForWrite("work", job);
-      FileSystem lfs = FileSystem.getLocal(job).getRaw();
+      workDir = lDirAlloc.getLocalPathForWrite("work", conf);
+      FileSystem lfs = FileSystem.getLocal(conf).getRaw();
       boolean madeDir = false;
       try {
         madeDir = lfs.mkdirs(workDir);
@@ -281,155 +241,29 @@ public class YarnTezDagChild {
         // at the same time. If this task loses the race, it's okay because
         // the directory already exists.
         madeDir = true;
-        workDir = lDirAlloc.getLocalPathToRead("work", job);
+        workDir = lDirAlloc.getLocalPathToRead("work", conf);
       }
       if (!madeDir) {
           throw new IOException("Mkdirs failed to create "
               + workDir.toString());
       }
     }
-    // TODO TEZ This likely needs fixing to make sure things work when there are multiple local-dirs etc.
-    job.set(MRJobConfig.JOB_LOCAL_DIR, workDir.toString());
-  }
-
-  private static JobConf configureTask(MRTask task, Credentials credentials,
-      Token<JobTokenIdentifier> jt, int appAttemptId)
-          throws IOException, InterruptedException {
-    JobConf job = task.getConf();
-    
-    // Set it in conf, so as to be able to be used the the OutputCommitter.
-    job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId);
-
-    // set tcp nodelay
-    job.setBoolean("ipc.client.tcpnodelay", true);
-    job.setClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
-        YarnOutputFiles.class, MapOutputFile.class);
-    // set the jobTokenFile into task
-    SecretKey sk = JobTokenSecretManager.createSecretKey(jt.getPassword());
-
-    task.setJobTokenSecret(sk);
-//    task.setJobTokenSecret(
-//        JobTokenSecretManager.createSecretKey(jt.getPassword()));
-
-    // setup the child's MRConfig.LOCAL_DIR.
-    configureLocalDirs(job);
-
-    // setup the child's attempt directories
-    // Do the task-type specific localization
-    task.localizeConfiguration(job);
-
-    // Set up the DistributedCache related configs
-    setupDistributedCacheConfig(job);
-
-    // Overwrite the localized task jobconf which is linked to in the current
-    // work-dir.
-    Path localTaskFile = new Path(MRJobConfig.JOB_CONF_FILE);
-    writeLocalJobFile(localTaskFile, job);
-    task.setConf(job);
-    return job;
-  }
-
-  /**
-   * Set up the DistributedCache related configs to make
-   * {@link DistributedCache#getLocalCacheFiles(Configuration)}
-   * and
-   * {@link DistributedCache#getLocalCacheArchives(Configuration)}
-   * working.
-   * @param job
-   * @throws IOException
-   */
-  private static void setupDistributedCacheConfig(final JobConf job)
-      throws IOException {
-
-    String localWorkDir = System.getenv("PWD");
-    //        ^ ^ all symlinks are created in the current work-dir
-
-    // Update the configuration object with localized archives.
-    URI[] cacheArchives = DistributedCache.getCacheArchives(job);
-    if (cacheArchives != null) {
-      List<String> localArchives = new ArrayList<String>();
-      for (int i = 0; i < cacheArchives.length; ++i) {
-        URI u = cacheArchives[i];
-        Path p = new Path(u);
-        Path name =
-            new Path((null == u.getFragment()) ? p.getName()
-                : u.getFragment());
-        String linkName = name.toUri().getPath();
-        localArchives.add(new Path(localWorkDir, linkName).toUri().getPath());
-      }
-      if (!localArchives.isEmpty()) {
-        job.set(MRJobConfig.CACHE_LOCALARCHIVES, StringUtils
-            .arrayToString(localArchives.toArray(new String[localArchives
-                .size()])));
-      }
-    }
-
-    // Update the configuration object with localized files.
-    URI[] cacheFiles = DistributedCache.getCacheFiles(job);
-    if (cacheFiles != null) {
-      List<String> localFiles = new ArrayList<String>();
-      for (int i = 0; i < cacheFiles.length; ++i) {
-        URI u = cacheFiles[i];
-        Path p = new Path(u);
-        Path name =
-            new Path((null == u.getFragment()) ? p.getName()
-                : u.getFragment());
-        String linkName = name.toUri().getPath();
-        localFiles.add(new Path(localWorkDir, linkName).toUri().getPath());
-      }
-      if (!localFiles.isEmpty()) {
-        job.set(MRJobConfig.CACHE_LOCALFILES,
-            StringUtils.arrayToString(localFiles
-                .toArray(new String[localFiles.size()])));
-      }
-    }
-  }
-
-  private static final FsPermission urw_gr =
-    FsPermission.createImmutable((short) 0640);
-
-  /**
-   * Write the task specific job-configuration file.
-   * @throws IOException
-   */
-  private static void writeLocalJobFile(Path jobFile, JobConf conf)
-      throws IOException {
-    FileSystem localFs = FileSystem.getLocal(conf);
-    localFs.delete(jobFile);
-    OutputStream out = null;
-    try {
-      out = FileSystem.create(localFs, jobFile, urw_gr);
-      conf.writeXml(out);
-    } finally {
-      IOUtils.cleanup(LOG, out);
-    }
+    conf.set(TezJobConfig.JOB_LOCAL_DIR, workDir.toString());
   }
-
+  
   private static Task createAndConfigureTezTask(
-      TezEngineTaskContext taskContext,
-      TezTaskUmbilicalProtocol master,
-      Credentials credentials, Token<JobTokenIdentifier> jt,
-      int appAttemptId)
-      throws IOException, InterruptedException {
+      TezEngineTaskContext taskContext, TezTaskUmbilicalProtocol master,
+      Credentials cxredentials, Token<JobTokenIdentifier> jobToken,
+      int appAttemptId) throws IOException, InterruptedException {
 
-    Configuration jConf = new JobConf(MRJobConfig.JOB_CONF_FILE);
-    Configuration conf = MultiStageMRConfigUtil.getConfForVertex(jConf,
-        taskContext.getVertexName());
+    Configuration conf = new Configuration();
+    // set tcp nodelay
+    conf.setBoolean("ipc.client.tcpnodelay", true);
+    conf.setInt(TezJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId);
+    
+    configureLocalDirs(conf);
+    
     
-    // TOOD Post MRR
-    // A single file per vertex will likely be a better solution. Does not
-    // require translation - client can take care of this. Will work independent
-    // of whether the configuration is for intermediate tasks or not. Has the
-    // overhead of localizing multiple files per job - i.e. the client would
-    // need to write these files to hdfs, add them as local resources per
-    // vertex. A solution like this may be more practical once it's possible to
-    // submit configuration parameters to the AM and effectively tasks via RPC.
-
-    // TODO Avoid all this extra config manipulation.
-    // FIXME we need I/O/p level configs to be used in init below
-    final JobConf job = new JobConf(conf);
-    job.setCredentials(credentials);
-
     // FIXME need Input/Output vertices else we have this hack
     if (taskContext.getInputSpecList().isEmpty()) {
       taskContext.getInputSpecList().add(
@@ -440,26 +274,37 @@ public class YarnTezDagChild {
           new OutputSpec("null", 0, SimpleOutput.class.getName()));
     }
     Task t = RuntimeUtils.createRuntimeTask(taskContext);
-    
+    t.initialize(conf, master);
+
     // FIXME wrapper should initialize all of processor, inputs and outputs
     // Currently, processor is inited via task init
     // and processor then inits inputs and outputs
-    t.initialize(job, master);
-    
-    MRTask task = (MRTask)t.getProcessor();
-    configureTask(task, credentials, jt, appAttemptId);
-    
     return t;
   }
   
   private static void runTezTask(
-      Task t, TezTaskUmbilicalProtocol master, JobConf job) 
+      Task t, TezTaskUmbilicalProtocol master, Configuration conf) 
   throws IOException, InterruptedException {
     // use job-specified working directory
-    FileSystem.get(job).setWorkingDirectory(job.getWorkingDirectory());
+    FileSystem.get(conf).setWorkingDirectory(getWorkingDirectory(conf));
     
     // Run!
     t.run();
     t.close();
   }
+  
+  private static Path getWorkingDirectory(Configuration conf) {
+    String name = conf.get(JobContext.WORKING_DIR);
+    if (name != null) {
+      return new Path(name);
+    } else {
+      try {
+        Path dir = FileSystem.get(conf).getWorkingDirectory();
+        conf.set(JobContext.WORKING_DIR, dir.toString());
+        return dir;
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine-api/src/main/java/org/apache/tez/engine/api/Task.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/api/Task.java b/tez-engine-api/src/main/java/org/apache/tez/engine/api/Task.java
index ea50a06..c08eda6 100644
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/api/Task.java
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/api/Task.java
@@ -74,4 +74,6 @@ public interface Task {
    */
   public void close() throws IOException, InterruptedException;
   
+  
+  public Configuration getConfiguration();
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
index 5823de6..93aeb0b 100644
--- a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
+++ b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
@@ -55,6 +55,11 @@ public class TezEngineTaskContext extends TezTaskContext {
     this.processorName = processorName;
   }
 
+  public String getRuntimeName() {
+    // FIXME. Add this to the DAG configuration, and fetch from there.
+    return "org.apache.tez.mapreduce.task.MRRuntimeTask";
+  }
+
   public String getProcessorName() {
     return processorName;
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java
index 6a065de..202abd6 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java
@@ -80,7 +80,7 @@ public class Shuffle implements ExceptionReporter {
     
     FileSystem localFS = FileSystem.getLocal(this.conf);
     LocalDirAllocator localDirAllocator = 
-        new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
+        new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
     
     copyPhase = this.runningTaskContext.getProgress().addPhase("copy", 0.33f);
     mergePhase = this.runningTaskContext.getProgress().addPhase("merge", 0.66f);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/TezMerger.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/TezMerger.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/TezMerger.java
index 829495b..eeea764 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/TezMerger.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/TezMerger.java
@@ -57,7 +57,7 @@ public class TezMerger {
 
   // Local directories
   private static LocalDirAllocator lDirAlloc = 
-    new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
+    new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
 
   public static
   TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
index ccfbd78..69484af 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
@@ -42,7 +42,7 @@ import org.apache.tez.dag.records.TezTaskID;
 public class TezLocalTaskOutputFiles extends TezTaskOutput {
 
   private LocalDirAllocator lDirAlloc =
-    new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
+    new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
 
   public TezLocalTaskOutputFiles() {
   }
@@ -223,7 +223,7 @@ public class TezLocalTaskOutputFiles extends TezTaskOutput {
   }
 
   private String[] getLocalDirs() throws IOException {
-    return getConf().getStrings(TezJobConfig.LOCAL_DIR);
+    return getConf().getStrings(TezJobConfig.LOCAL_DIRS);
   }
 
   @SuppressWarnings("deprecation")

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
index 1bd65e2..5fb6519 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
@@ -56,7 +56,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
 
   // assume configured to $localdir/usercache/$user/appcache/$appId
   private LocalDirAllocator lDirAlloc = 
-    new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
+    new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
 
   private Path getAttemptOutputDir() {
     LOG.info("DEBUG: getAttemptOutputDir: "

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
index e04d405..900c2f0 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
@@ -66,6 +66,7 @@ public class RuntimeUtils {
         + ", Processor: " + taskContext.getProcessorName()
         + ", InputCount=" + taskContext.getInputSpecList().size()
         + ", OutputCount=" + taskContext.getOutputSpecList().size());
+
     RuntimeTask t = null;
     try {
       Class<?> processorClazz =
@@ -108,11 +109,30 @@ public class RuntimeUtils {
           outputs[i] = output;
         }
       }
-      t = new RuntimeTask(processor, inputs, outputs);
+      // t = new RuntimeTask(taskContext, processor, inputs, outputs);
+      t = createRuntime(taskContext, processor, inputs, outputs);
     } catch (ClassNotFoundException e) {
       throw new YarnException("Unable to initialize RuntimeTask, context="
           + taskContext, e);
     }
     return t;
   }
+  
+  private static RuntimeTask createRuntime(TezEngineTaskContext taskContext,
+      Processor processor, Input[] inputs, Output[] outputs) {
+    try {
+      // TODO Change this to use getNewInstance
+      Class<?> runtimeClazz = Class.forName(taskContext.getRuntimeName());
+      Constructor<?> ctor = runtimeClazz.getConstructor(
+          TezEngineTaskContext.class, Processor.class, Input[].class,
+          Output[].class);
+      ctor.setAccessible(true);
+      return (RuntimeTask) ctor.newInstance(taskContext, processor, inputs, outputs);
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException("Unable to load runtimeClass: "
+          + taskContext.getRuntimeName(), e);
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/task/RuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/task/RuntimeTask.java b/tez-engine/src/main/java/org/apache/tez/engine/task/RuntimeTask.java
index 3e2c6f2..871f3ba 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/task/RuntimeTask.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/task/RuntimeTask.java
@@ -20,6 +20,7 @@ package org.apache.tez.engine.task;
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.TezEngineTaskContext;
 import org.apache.tez.engine.api.Input;
 import org.apache.tez.engine.api.Master;
 import org.apache.tez.engine.api.Output;
@@ -28,17 +29,19 @@ import org.apache.tez.engine.api.Task;
 
 public class RuntimeTask implements Task {
 
-  private final Input[] inputs;
-  private final Output[] outputs;
-  private final Processor processor;
+  protected final Input[] inputs;
+  protected final Output[] outputs;
+  protected final Processor processor;
   
-  private Configuration conf;
-  private Master master;
+  protected TezEngineTaskContext taskContext;
+  protected Configuration conf;
+  protected Master master;
   
-  public RuntimeTask(
+  public RuntimeTask(TezEngineTaskContext taskContext,
       Processor processor,
       Input[] inputs,
       Output[] outputs) {
+    this.taskContext = taskContext;
     this.inputs = inputs;
     this.processor = processor;
     this.outputs = outputs;
@@ -74,7 +77,13 @@ public class RuntimeTask implements Task {
 
   public void close() throws IOException, InterruptedException {
     // NOTE: Allow processor to close input/output
+    // This can be changed to close input/output since MRRuntimeTask is used for
+    // MR jobs, which changes the order.
     processor.close();
   }
 
+  @Override
+  public Configuration getConfiguration() {
+    return this.conf;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 63472d2..5c181e2 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -465,7 +465,7 @@ public class LocalJobRunner implements ClientProtocol {
                 LOG.info("XXX mapId: " + i + 
                     " LOCAL_DIR = " + 
                     mapOutputFiles.get(mapId).getConf().get(
-                        TezJobConfig.LOCAL_DIR));
+                        TezJobConfig.LOCAL_DIRS));
                 Path mapOut = mapOutputFiles.get(mapId).getOutputFile();
                 TezTaskOutput localOutputFile = new TezLocalTaskOutputFiles();
                 localOutputFile.setConf(localConf);
@@ -902,7 +902,7 @@ public class LocalJobRunner implements ClientProtocol {
       TaskAttemptID taskAttemptID, String user, JobConf conf) {
     String[] localDirs = 
         conf.getTrimmedStrings(
-            TezJobConfig.LOCAL_DIR, TezJobConfig.DEFAULT_LOCAL_DIR);
+            TezJobConfig.LOCAL_DIRS, TezJobConfig.DEFAULT_LOCAL_DIRS);
     String jobId = taskAttemptID.getJobID().toString();
     String taskId = taskAttemptID.getTaskID().toString();
     boolean isCleanup = false;
@@ -913,9 +913,9 @@ public class LocalJobRunner implements ClientProtocol {
       childMapredLocalDir.append("," + localDirs[i] + Path.SEPARATOR
           + getLocalTaskDir(user, jobId, taskId, isCleanup));
     }
-    LOG.info(TezJobConfig.LOCAL_DIR + " for child : " + taskAttemptID + 
+    LOG.info(TezJobConfig.LOCAL_DIRS + " for child : " + taskAttemptID + 
         " is " + childMapredLocalDir);
-    conf.set(TezJobConfig.LOCAL_DIR, childMapredLocalDir.toString());
+    conf.set(TezJobConfig.LOCAL_DIRS, childMapredLocalDir.toString());
     conf.setClass(Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER, 
         TezLocalTaskOutputFiles.class, TezTaskOutput.class);
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
new file mode 100644
index 0000000..6091d4b
--- /dev/null
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -0,0 +1,262 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.task;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.crypto.SecretKey;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSError;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapOutputFile;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.filecache.DistributedCache;
+import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
+import org.apache.tez.common.TezEngineTaskContext;
+import org.apache.tez.common.TezJobConfig;
+import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.engine.api.Input;
+import org.apache.tez.engine.api.Master;
+import org.apache.tez.engine.api.Output;
+import org.apache.tez.engine.api.Processor;
+import org.apache.tez.engine.common.security.JobTokenIdentifier;
+import org.apache.tez.engine.common.security.TokenCache;
+import org.apache.tez.engine.task.RuntimeTask;
+import org.apache.tez.mapreduce.hadoop.DeprecatedKeys;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+import org.apache.tez.mapreduce.processor.MRTask;
+import org.apache.tez.mapreduce.task.impl.YarnOutputFiles;
+
+@SuppressWarnings("deprecation")
+public class MRRuntimeTask extends RuntimeTask {
+
+  private static final Log LOG = LogFactory.getLog(MRRuntimeTask.class);
+
+  private MRTask mrTask;
+
+  public MRRuntimeTask(TezEngineTaskContext taskContext, Processor processor,
+      Input[] inputs, Output[] outputs) {
+    super(taskContext, processor, inputs, outputs);
+  }
+
+  @Override
+  public void initialize(Configuration conf, Master master) throws IOException,
+      InterruptedException {
+
+    DeprecatedKeys.init();
+
+    Configuration mrConf = new Configuration(conf);
+    mrConf.addResource(MRJobConfig.JOB_CONF_FILE);
+    Configuration taskConf = MultiStageMRConfigUtil.getConfForVertex(mrConf,
+        taskContext.getVertexName());
+
+    // TODO Avoid all this extra config manipulation.
+    // FIXME we need I/O/p level configs to be used in init below
+
+    // TOOD Post MRR
+    // A single file per vertex will likely be a better solution. Does not
+    // require translation - client can take care of this. Will work independent
+    // of whether the configuration is for intermediate tasks or not. Has the
+    // overhead of localizing multiple files per job - i.e. the client would
+    // need to write these files to hdfs, add them as local resources per
+    // vertex. A solution like this may be more practical once it's possible to
+    // submit configuration parameters to the AM and effectively tasks via RPC.
+
+    final JobConf job = new JobConf(taskConf);
+
+    MRTask mrTask = (MRTask) getProcessor();
+    this.mrTask = mrTask;
+    configureMRTask(job, mrTask);
+
+    this.conf = job;
+    this.master = master;
+
+    // NOTE: Allow processor to initialize input/output
+    processor.initialize(this.conf, this.master);
+  }
+
+  @Override
+  public void run() throws IOException, InterruptedException {
+    TezTaskUmbilicalProtocol umbilical = (TezTaskUmbilicalProtocol) master;
+    try {
+      super.run();
+    } catch (FSError e) {
+      throw e;
+    } catch (Exception exception) {
+      LOG.warn("Exception running child : "
+          + StringUtils.stringifyException(exception));
+      try {
+        if (mrTask != null) {
+          mrTask.taskCleanup(umbilical);
+        }
+      } catch (Exception e) {
+        LOG.info("Exception cleanup up: " + StringUtils.stringifyException(e));
+      }
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      exception.printStackTrace(new PrintStream(baos));
+      if (taskContext.getTaskAttemptId() != null) {
+        umbilical.fatalError(taskContext.getTaskAttemptId(), baos.toString());
+      }
+    }
+  }
+
+  @Override
+  public void close() throws IOException, InterruptedException {
+    // NOTE: Allow processor to close input/output
+    processor.close();
+  }
+
+  private static void configureMRTask(JobConf job, MRTask task)
+      throws IOException, InterruptedException {
+
+    Credentials credentials = UserGroupInformation.getCurrentUser()
+        .getCredentials();
+    job.setCredentials(credentials);
+    // TODO Can this be avoided all together. Have the MRTezOutputCommitter use
+    // the Tez parameter.
+    // TODO This could be fetched from the env if YARN is setting it for all
+    // Containers.
+    // Set it in conf, so as to be able to be used the the OutputCommitter.
+    job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
+        job.getInt(TezJobConfig.APPLICATION_ATTEMPT_ID, -1));
+
+    job.setClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS, YarnOutputFiles.class,
+        MapOutputFile.class); // MR
+
+    Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(credentials);
+    if (jobToken != null) {
+      // Will MR ever run without a job token.
+      SecretKey sk = JobTokenSecretManager.createSecretKey(jobToken
+          .getPassword());
+      task.setJobTokenSecret(sk);
+    } else {
+      LOG.warn("No job token set");
+    }
+
+    job.set(MRJobConfig.JOB_LOCAL_DIR, job.get(TezJobConfig.JOB_LOCAL_DIR));
+
+    // setup the child's attempt directories
+    // Do the task-type specific localization
+    task.localizeConfiguration(job);
+
+    // Set up the DistributedCache related configs
+    setupDistributedCacheConfig(job);
+
+    // Overwrite the localized task jobconf which is linked to in the current
+    // work-dir.
+    Path localTaskFile = new Path(
+        job.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR),
+        MRJobConfig.JOB_CONF_FILE);
+    writeLocalJobFile(localTaskFile, job);
+
+    task.setConf(job);
+  }
+
+  /**
+   * Set up the DistributedCache related configs to make
+   * {@link DistributedCache#getLocalCacheFiles(Configuration)} and
+   * {@link DistributedCache#getLocalCacheArchives(Configuration)} working.
+   * 
+   * @param job
+   * @throws IOException
+   */
+  private static void setupDistributedCacheConfig(final JobConf job)
+      throws IOException {
+
+    String localWorkDir = System.getenv(System.getenv(Environment.PWD.name()));
+    // ^ ^ all symlinks are created in the current work-dir
+
+    // Update the configuration object with localized archives.
+    URI[] cacheArchives = DistributedCache.getCacheArchives(job);
+    if (cacheArchives != null) {
+      List<String> localArchives = new ArrayList<String>();
+      for (int i = 0; i < cacheArchives.length; ++i) {
+        URI u = cacheArchives[i];
+        Path p = new Path(u);
+        Path name = new Path((null == u.getFragment()) ? p.getName()
+            : u.getFragment());
+        String linkName = name.toUri().getPath();
+        localArchives.add(new Path(localWorkDir, linkName).toUri().getPath());
+      }
+      if (!localArchives.isEmpty()) {
+        job.set(MRJobConfig.CACHE_LOCALARCHIVES, StringUtils
+            .arrayToString(localArchives.toArray(new String[localArchives
+                .size()])));
+      }
+    }
+
+    // Update the configuration object with localized files.
+    URI[] cacheFiles = DistributedCache.getCacheFiles(job);
+    if (cacheFiles != null) {
+      List<String> localFiles = new ArrayList<String>();
+      for (int i = 0; i < cacheFiles.length; ++i) {
+        URI u = cacheFiles[i];
+        Path p = new Path(u);
+        Path name = new Path((null == u.getFragment()) ? p.getName()
+            : u.getFragment());
+        String linkName = name.toUri().getPath();
+        localFiles.add(new Path(localWorkDir, linkName).toUri().getPath());
+      }
+      if (!localFiles.isEmpty()) {
+        job.set(MRJobConfig.CACHE_LOCALFILES, StringUtils
+            .arrayToString(localFiles.toArray(new String[localFiles.size()])));
+      }
+    }
+  }
+
+  private static final FsPermission urw_gr = FsPermission
+      .createImmutable((short) 0640);
+
+  /**
+   * Write the task specific job-configuration file.
+   * 
+   * @throws IOException
+   */
+  private static void writeLocalJobFile(Path jobFile, JobConf conf)
+      throws IOException {
+    FileSystem localFs = FileSystem.getLocal(conf);
+    localFs.delete(jobFile);
+    OutputStream out = null;
+    try {
+      out = FileSystem.create(localFs, jobFile, urw_gr);
+      conf.writeXml(out);
+    } finally {
+      IOUtils.cleanup(LOG, out);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
index 0b6bc5f..48a1113 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
@@ -24,14 +24,17 @@ import java.util.Random;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
 import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.FileSplit;
 import org.apache.hadoop.mapred.InputSplit;
@@ -40,6 +43,8 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapreduce.split.JobSplit;
 import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo;
 import org.apache.hadoop.mapreduce.split.SplitMetaInfoReaderTez;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.tez.common.InputSpec;
 import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.TezEngineTaskContext;
@@ -55,6 +60,51 @@ public class MapUtils {
 
   private static final Log LOG = LogFactory.getLog(MapUtils.class);
   
+  public static void configureLocalDirs(Configuration conf, String localDir)
+      throws IOException {
+    String[] localSysDirs = new String[1];
+    localSysDirs[0] = localDir;
+
+    conf.setStrings(TezJobConfig.LOCAL_DIRS, localSysDirs);
+    conf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
+        System.getenv(Environment.PWD.name()));
+
+    LOG.info(TezJobConfig.LOCAL_DIRS + " for child: "
+        + conf.get(TezJobConfig.LOCAL_DIRS));
+    LOG.info(TezJobConfig.TASK_LOCAL_RESOURCE_DIR + " for child: "
+        + conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
+
+    LocalDirAllocator lDirAlloc = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
+    Path workDir = null;
+    // First, try to find the JOB_LOCAL_DIR on this host.
+    try {
+      workDir = lDirAlloc.getLocalPathToRead("work", conf);
+    } catch (DiskErrorException e) {
+      // DiskErrorException means dir not found. If not found, it will
+      // be created below.
+    }
+    if (workDir == null) {
+      // JOB_LOCAL_DIR doesn't exist on this host -- Create it.
+      workDir = lDirAlloc.getLocalPathForWrite("work", conf);
+      FileSystem lfs = FileSystem.getLocal(conf).getRaw();
+      boolean madeDir = false;
+      try {
+        madeDir = lfs.mkdirs(workDir);
+      } catch (FileAlreadyExistsException e) {
+        // Since all tasks will be running in their own JVM, the race condition
+        // exists where multiple tasks could be trying to create this directory
+        // at the same time. If this task loses the race, it's okay because
+        // the directory already exists.
+        madeDir = true;
+        workDir = lDirAlloc.getLocalPathToRead("work", conf);
+      }
+      if (!madeDir) {
+        throw new IOException("Mkdirs failed to create " + workDir.toString());
+      }
+    }
+    conf.set(TezJobConfig.JOB_LOCAL_DIR, workDir.toString());
+  }
+  
   private static InputSplit 
   createInputSplit(FileSystem fs, Path workDir, JobConf job, Path file) 
       throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
index cda15fb..1e5fe9b 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
@@ -55,36 +55,40 @@ import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 
+@SuppressWarnings("deprecation")
 public class TestMapProcessor {
   
   private static final Log LOG = LogFactory.getLog(TestMapProcessor.class);  
   
   private static JobConf defaultConf = new JobConf();
   private static FileSystem localFs = null; 
+  private static Path workDir = null;
   static {
     try {
       defaultConf.set("fs.defaultFS", "file:///");
       localFs = FileSystem.getLocal(defaultConf);
+      workDir =
+          new Path(new Path(System.getProperty("test.build.data", "/tmp")),
+                   "TestMapProcessor").makeQualified(localFs);
+      MapUtils.configureLocalDirs(defaultConf, workDir.toString());
     } catch (IOException e) {
       throw new RuntimeException("init failure", e);
     }
   }
-  @SuppressWarnings("deprecation")
-  private static Path workDir =
-    new Path(new Path(System.getProperty("test.build.data", "/tmp")),
-             "TestMapProcessor").makeQualified(localFs);
+  
+
 
   TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
 
   public void setUpJobConf(JobConf job) {
-    job.set(TezJobConfig.LOCAL_DIR, workDir.toString());
+    job.set(TezJobConfig.LOCAL_DIRS, workDir.toString());
     job.setClass(
         Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
         TezLocalTaskOutputFiles.class, 
         TezTaskOutput.class);
     job.setNumReduceTasks(1);
   }
-  
+
   @Before
   @After
   public void cleanup() throws Exception {
@@ -98,15 +102,18 @@ public class TestMapProcessor {
     setUpJobConf(jobConf);
     TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
     mapOutputs.setConf(jobConf);
-    
+
     Configuration conf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
+    conf.setInt(TezJobConfig.APPLICATION_ATTEMPT_ID, 0);
+
     Configuration stageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
         vertexName);
     
     JobConf job = new JobConf(stageConf);
-    
+
     job.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
         "localized-resources").toUri().toString());
+    
 
     MapUtils.runMapProcessor(localFs, workDir, job, 0,
         new Path(workDir, "map0"), new TestUmbilicalProtocol(), vertexName,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
index 69571e1..d17b0be 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
@@ -56,27 +56,30 @@ import org.junit.Before;
 import org.junit.Test;
 
 
+@SuppressWarnings("deprecation")
 public class TestReduceProcessor {
   
   private static final Log LOG = LogFactory.getLog(TestReduceProcessor.class);
 
   private static JobConf defaultConf = new JobConf();
-  private static FileSystem localFs = null; 
+  private static FileSystem localFs = null;
+  private static Path workDir = null;
   static {
     try {
       defaultConf.set("fs.defaultFS", "file:///");
       localFs = FileSystem.getLocal(defaultConf);
+      workDir =
+          new Path(new Path(System.getProperty("test.build.data", "/tmp")),
+                   "TestReduceProcessor").makeQualified(localFs);
+      
+      MapUtils.configureLocalDirs(defaultConf, workDir.toString());
     } catch (IOException e) {
       throw new RuntimeException("init failure", e);
     }
   }
-  @SuppressWarnings("deprecation")
-  private static Path workDir =
-    new Path(new Path(System.getProperty("test.build.data", "/tmp")),
-             "TestReduceProcessor").makeQualified(localFs);
 
   public void setUpJobConf(JobConf job) {
-    job.set(TezJobConfig.LOCAL_DIR, workDir.toString());
+    job.set(TezJobConfig.LOCAL_DIRS, workDir.toString());
     job.setClass(
         Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
         TezLocalTaskOutputFiles.class, 
@@ -101,6 +104,7 @@ public class TestReduceProcessor {
     mapOutputs.setConf(jobConf);
     
     Configuration conf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
+    conf.setInt(TezJobConfig.APPLICATION_ATTEMPT_ID, 0);
     Configuration mapStageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
         mapVertexName);
     
@@ -109,6 +113,7 @@ public class TestReduceProcessor {
     mapConf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
         "localized-resources").toUri().toString());
     
+    
     // Run a map
     MapUtils.runMapProcessor(localFs, workDir, mapConf, 0,
         new Path(workDir, "map0"), new TestUmbilicalProtocol(), mapVertexName,
@@ -124,6 +129,8 @@ public class TestReduceProcessor {
         reduceVertexName);
     JobConf reduceConf = new JobConf(reduceStageConf);
     reduceConf.setOutputFormat(SequenceFileOutputFormat.class);
+    reduceConf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
+        "localized-resources").toUri().toString());
     FileOutputFormat.setOutputPath(reduceConf, new Path(workDir, "output"));
     
     // Now run a reduce
@@ -147,8 +154,8 @@ public class TestReduceProcessor {
             .toMRTaskId(taskContext.getTaskAttemptId().getTaskID()));
     Path reduceOutputFile = new Path(reduceOutputDir, "part-00000");
 
-    @SuppressWarnings("deprecation")
-    SequenceFile.Reader reader = new SequenceFile.Reader(localFs, reduceOutputFile, reduceConf);
+    SequenceFile.Reader reader = new SequenceFile.Reader(localFs,
+        reduceOutputFile, reduceConf);
 
     LongWritable key = new LongWritable();
     Text value = new Text();


[26/50] [abbrv] git commit: TEZ-150. Fix usage of mapreduce API keys for num maps and num reduces. Removes Tez configuration for in-degree and out-degree. (sseth)

Posted by ss...@apache.org.
TEZ-150. Fix usage of mapreduce API keys for num maps and num reduces.
Removes Tez configuration for in-degree and out-degree. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/4c2732c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/4c2732c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/4c2732c9

Branch: refs/heads/master
Commit: 4c2732c991a4cc76011acedbf9c17726c06bc145
Parents: 268f3ac
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu May 23 13:40:22 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu May 23 13:40:22 2013 -0700

----------------------------------------------------------------------
 .../java/org/apache/tez/common/TezJobConfig.java   |   16 +---------
 .../org/apache/hadoop/mapred/LocalJobRunner.java   |    1 -
 .../tez/mapreduce/hadoop/DeprecatedKeys.java       |   17 ++++------
 .../processor/reduce/ReduceProcessor.java          |    1 -
 .../apache/tez/mapreduce/task/MRRuntimeTask.java   |    4 ++
 .../tez/mapreduce/hadoop/TestDeprecatedKeys.java   |   23 +-------------
 6 files changed, 14 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
index 0d6f6be..c0b0ae6 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -66,7 +66,7 @@ public class TezJobConfig {
    */
   public static final String LOCAL_DIRS = "tez.engine.local.dirs";
   public static final String DEFAULT_LOCAL_DIRS = "/tmp";
-  
+
   /**
    * One local dir for the speicfic job.
    */
@@ -80,20 +80,6 @@ public class TezJobConfig {
   public static final String DEFAULT_TASK_LOCAL_RESOURCE_DIR = "/tmp";
   
   public static final String TEZ_TASK_WORKING_DIR = "tez.engine.task.working.dir";
-  
-  /**
-   * 
-   */
-  public static final String TEZ_ENGINE_TASK_INDEGREE = 
-      "tez.engine.task.in-degree";
-  public static final int DEFAULT_TEZ_ENGINE_TASK_INDEGREE = 1;
-
-  /**
-   * 
-   */
-  public static final String TEZ_ENGINE_TASK_OUTDEGREE = 
-      "tez.engine.task.out-degree";
-  public static final int DEFAULT_TEZ_ENGINE_TASK_OUTDEGREE = 1;
 
   /**
    * 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 5c181e2..5dade2b 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -445,7 +445,6 @@ public class LocalJobRunner implements ClientProtocol {
             JobConf localConf = new JobConf(job);
             localConf.setUser(user);
             localConf.set("mapreduce.jobtracker.address", "local");
-            localConf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, mapIds.size());
             setupChildMapredLocalDirs(reduceId, user, localConf);
 
             // FIXME invalid task context

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index 01d4fe0..499159f 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -136,8 +136,6 @@ public class DeprecatedKeys {
     registerMRToEngineKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD_BYTES, TezJobConfig.TEZ_ENGINE_IFILE_READAHEAD_BYTES);
     
     registerMRToEngineKeyTranslation(MRJobConfig.RECORDS_BEFORE_PROGRESS, TezJobConfig.RECORDS_BEFORE_PROGRESS);
-    
-    registerMRToEngineKeyTranslation(MRJobConfig.JOB_LOCAL_DIR, MRConfig.LOCAL_DIR);
 
     registerMRToEngineKeyTranslation(MRJobConfig.IO_SORT_FACTOR, TezJobConfig.TEZ_ENGINE_IO_SORT_FACTOR);
     
@@ -149,6 +147,12 @@ public class DeprecatedKeys {
     
     registerMRToEngineKeyTranslation(MRJobConfig.MAP_COMBINE_MIN_SPILLS, TezJobConfig.TEZ_ENGINE_COMBINE_MIN_SPILLS);
     
+    // Counter replacement will work in this manner, as long as TezCounters
+    // extends MRCounters and is used directly by the Mapper/Reducer.
+    // When these counters are eventually translated over to MRCounters, this
+    // may break.
+    // Framework counters, like FILESYSTEM will likely be incompatible since
+    // they enum key belongs to a different package.
     registerMRToEngineKeyTranslation(MRJobConfig.COUNTERS_MAX_KEY, TezJobConfig.COUNTERS_MAX_KEY);
     
     registerMRToEngineKeyTranslation(MRJobConfig.COUNTER_GROUP_NAME_MAX_KEY, TezJobConfig.COUNTER_GROUP_NAME_MAX_KEY);
@@ -180,7 +184,7 @@ public class DeprecatedKeys {
     registerMRToEngineKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, TezJobConfig.TEZ_ENGINE_SHUFFLE_ENABLE_MEMTOMEM);
     
     registerMRToEngineKeyTranslation(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, TezJobConfig.TEZ_ENGINE_INPUT_BUFFER_PERCENT);
-    
+
     registerMRToEngineKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezJobConfig.DAG_CREDENTIALS_BINARY);
     
     registerMRToEngineKeyTranslation("map.sort.class", TezJobConfig.TEZ_ENGINE_INTERNAL_SORTER_CLASS);
@@ -188,17 +192,10 @@ public class DeprecatedKeys {
     registerMRToEngineKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_ENGINE_GROUP_COMPARATOR_CLASS);
     
     registerMRToEngineKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_ENGINE_INTERMEDIATE_INPUT_KEY_SECONDARY_COMPARATOR_CLASS);
-    
-    registerMRToEngineKeyTranslation(MRJobConfig.NUM_REDUCES, TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE);
 
-    registerMRToEngineKeyTranslation(MRJobConfig.NUM_MAPS, TezJobConfig.TEZ_ENGINE_TASK_INDEGREE);
   }
   
   private static void addDeprecatedKeys() {
-    // Adding deprecation for num_maps, num_reduces - in case some component in
-    // Hadoop MR land uses these. Tez components already use tez-in/out-degree.
-    _(MRJobConfig.NUM_REDUCES, TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE);
-    _(MRJobConfig.NUM_MAPS, TezJobConfig.TEZ_ENGINE_TASK_INDEGREE);
   }
 
   private static void registerMRToEngineKeyTranslation(String mrKey,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
index ce49726..9a6fb5b 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
@@ -338,7 +338,6 @@ implements Processor {
       throws IOException, InterruptedException {
     super.localizeConfiguration(jobConf);
     jobConf.setBoolean(JobContext.TASK_ISMAP, false);
-    jobConf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, numMapTasks); 
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index 34aaa1b..a3d5684 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -170,6 +170,10 @@ public class MRRuntimeTask extends RuntimeTask {
 
     job.set(MRJobConfig.JOB_LOCAL_DIR, job.get(TezJobConfig.JOB_LOCAL_DIR));
     job.set(MRConfig.LOCAL_DIR, job.get(TezJobConfig.LOCAL_DIRS));
+    if (job.get(TezJobConfig.DAG_CREDENTIALS_BINARY) != null) {
+      job.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY,
+          job.get(TezJobConfig.DAG_CREDENTIALS_BINARY));
+    }
 
     // setup the child's attempt directories
     // Do the task-type specific localization

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
index 16e783e..99aad74 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
@@ -18,28 +18,9 @@
 
 package org.apache.tez.mapreduce.hadoop;
 
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.tez.common.TezJobConfig;
-import org.junit.Test;
 
 public class TestDeprecatedKeys {
 
-  @Test
-  public void testNumTasksDeprecation() {
-    
-    Configuration conf = new Configuration(false);
-    conf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, 6);
-    conf.setInt(TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE, 3);
-    
-    
-    DeprecatedKeys.init();
-    JobConf jobConf = new JobConf(conf);
-    
-    Assert.assertEquals(6, jobConf.getNumMapTasks());
-    Assert.assertEquals(3, jobConf.getNumReduceTasks());
-  }
-  
+  // TODO Add tests
+
 }


[08/50] [abbrv] git commit: TEZ-123. Cleanup package dependencies. (bikas)

Posted by ss...@apache.org.
TEZ-123. Cleanup package dependencies. (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/8193b5fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/8193b5fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/8193b5fd

Branch: refs/heads/master
Commit: 8193b5fddbff65e9e747f08ad36bc0ad41015ab0
Parents: 8aa2f30
Author: Bikas Saha <bi...@apache.org>
Authored: Mon May 13 17:35:57 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Mon May 13 17:35:57 2013 -0700

----------------------------------------------------------------------
 tez-common/pom.xml                                 |    4 -
 .../main/java/org/apache/tez/common/IDUtils.java   |   10 +-
 .../java/org/apache/tez/common/TezTaskContext.java |    4 +-
 .../org/apache/tez/common/TezTaskReporter.java     |  116 ------------
 .../java/org/apache/tez/common/TezTaskStatus.java  |    2 +-
 .../tez/common/TezTaskUmbilicalProtocol.java       |   79 ---------
 .../java/org/apache/tez/dag/records/TezDAGID.java  |  136 +++++++++++++++
 .../java/org/apache/tez/dag/records/TezID.java     |   97 ++++++++++
 .../apache/tez/dag/records/TezTaskAttemptID.java   |  128 ++++++++++++++
 .../java/org/apache/tez/dag/records/TezTaskID.java |  126 +++++++++++++
 .../org/apache/tez/dag/records/TezVertexID.java    |  129 ++++++++++++++
 .../org/apache/tez/dag/api/client/DAGStatus.java   |   39 ----
 .../apache/tez/dag/api/client/VertexStatus.java    |   51 ------
 tez-dag/pom.xml                                    |    4 +
 .../hadoop/mapred/MRVertexOutputCommitter.java     |    6 +-
 .../apache/hadoop/mapred/TezMRTypeConverter.java   |    8 +-
 .../org/apache/hadoop/mapred/YarnTezDagChild.java  |    2 +-
 .../tez/dag/api/client/impl/TezBuilderUtils.java   |    8 +-
 .../org/apache/tez/dag/api/impl/DAGStatus.java     |   39 ++++
 .../dag/api/impl/NullVertexOutputCommitter.java    |    2 +-
 .../org/apache/tez/dag/api/impl/VertexContext.java |    4 +-
 .../tez/dag/api/impl/VertexOutputCommitter.java    |    1 -
 .../org/apache/tez/dag/api/impl/VertexStatus.java  |   51 ++++++
 .../tez/dag/api/records/TaskAttemptReport.java     |    2 +-
 .../org/apache/tez/dag/api/records/TaskReport.java |    4 +-
 .../java/org/apache/tez/dag/app/AppContext.java    |    2 +-
 .../java/org/apache/tez/dag/app/DAGAppMaster.java  |    2 +-
 .../apache/tez/dag/app/TaskAttemptListener.java    |    2 +-
 .../tez/dag/app/TaskAttemptListenerImpTezDag.java  |    2 +-
 .../apache/tez/dag/app/TaskHeartbeatHandler.java   |    2 +-
 .../main/java/org/apache/tez/dag/app/dag/DAG.java  |    5 +-
 .../main/java/org/apache/tez/dag/app/dag/Task.java |    4 +-
 .../org/apache/tez/dag/app/dag/TaskAttempt.java    |    2 +-
 .../java/org/apache/tez/dag/app/dag/Vertex.java    |    4 +-
 .../apache/tez/dag/app/dag/VertexScheduler.java    |    2 +-
 .../org/apache/tez/dag/app/dag/event/DAGEvent.java |    2 +-
 .../dag/app/dag/event/DAGEventCounterUpdate.java   |    2 +-
 .../app/dag/event/DAGEventDiagnosticsUpdate.java   |    2 +-
 .../dag/app/dag/event/DAGEventVertexCompleted.java |    2 +-
 .../tez/dag/app/dag/event/DAGFinishEvent.java      |    2 +-
 .../tez/dag/app/dag/event/TaskAttemptEvent.java    |    2 +-
 .../event/TaskAttemptEventContainerTerminated.java |    2 +-
 .../TaskAttemptEventContainerTerminating.java      |    2 +-
 .../event/TaskAttemptEventDiagnosticsUpdate.java   |    2 +-
 .../app/dag/event/TaskAttemptEventFailRequest.java |    2 +-
 .../app/dag/event/TaskAttemptEventKillRequest.java |    2 +-
 .../app/dag/event/TaskAttemptEventNodeFailed.java  |    2 +-
 .../event/TaskAttemptEventOutputConsumable.java    |    2 +-
 .../app/dag/event/TaskAttemptEventSchedule.java    |    2 +-
 .../dag/event/TaskAttemptEventStartedRemotely.java |    2 +-
 .../dag/event/TaskAttemptEventStatusUpdate.java    |    2 +-
 .../apache/tez/dag/app/dag/event/TaskEvent.java    |    2 +-
 .../tez/dag/app/dag/event/TaskEventTAUpdate.java   |    2 +-
 .../apache/tez/dag/app/dag/event/VertexEvent.java  |    2 +-
 .../VertexEventSourceTaskAttemptCompleted.java     |    2 +-
 .../dag/event/VertexEventSourceVertexStarted.java  |    2 +-
 .../event/VertexEventTaskAttemptFetchFailure.java  |    2 +-
 .../app/dag/event/VertexEventTaskCompleted.java    |    2 +-
 .../app/dag/event/VertexEventTaskReschedule.java   |    2 +-
 .../impl/BipartiteSlowStartVertexScheduler.java    |    6 +-
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |    6 +-
 .../dag/impl/ImmediateStartVertexScheduler.java    |    2 +-
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java      |    4 +-
 .../org/apache/tez/dag/app/dag/impl/TaskImpl.java  |    6 +-
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |   12 +-
 .../tez/dag/app/rm/AMSchedulerEventTAEnded.java    |    2 +-
 .../app/rm/AMSchedulerEventTALaunchRequest.java    |    2 +-
 .../tez/dag/app/rm/container/AMContainer.java      |    2 +-
 .../app/rm/container/AMContainerEventAssignTA.java |    2 +-
 .../container/AMContainerEventLaunchRequest.java   |    4 +-
 .../rm/container/AMContainerEventTASucceeded.java  |    2 +-
 .../dag/app/rm/container/AMContainerHelpers.java   |    2 +-
 .../tez/dag/app/rm/container/AMContainerImpl.java  |    2 +-
 .../app/rm/node/AMNodeEventTaskAttemptEnded.java   |    2 +-
 .../rm/node/AMNodeEventTaskAttemptSucceeded.java   |    2 +-
 .../tez/dag/app/speculate/DefaultSpeculator.java   |    6 +-
 .../ExponentiallySmoothedTaskRuntimeEstimator.java |    2 +-
 .../app/speculate/LegacyTaskRuntimeEstimator.java  |    4 +-
 .../dag/app/speculate/NullTaskRuntimesEngine.java  |    4 +-
 .../tez/dag/app/speculate/SpeculatorEvent.java     |    6 +-
 .../tez/dag/app/speculate/StartEndTimesBase.java   |    6 +-
 .../dag/app/speculate/TaskRuntimeEstimator.java    |    4 +-
 .../app/speculate/TaskSpeculationPredicate.java    |    2 +-
 .../tez/dag/app/taskclean/TaskCleanupEvent.java    |    2 +-
 .../apache/tez/dag/history/DAGHistoryEvent.java    |    2 +-
 .../tez/dag/history/events/DAGFinishedEvent.java   |    4 +-
 .../tez/dag/history/events/DAGStartedEvent.java    |    2 +-
 .../history/events/TaskAttemptFinishedEvent.java   |    2 +-
 .../history/events/TaskAttemptStartedEvent.java    |    2 +-
 .../tez/dag/history/events/TaskFinishedEvent.java  |    2 +-
 .../tez/dag/history/events/TaskStartedEvent.java   |    2 +-
 .../dag/history/events/VertexFinishedEvent.java    |    4 +-
 .../tez/dag/history/events/VertexStartedEvent.java |    2 +-
 .../apache/tez/dag/utils/TezEngineChildJVM.java    |    3 +-
 .../tez/dag/app/dag/impl/TestTaskAttempt.java      |    8 +-
 .../apache/tez/dag/app/dag/impl/TestTaskImpl.java  |    8 +-
 tez-engine-api/pom.xml                             |    4 +
 .../java/org/apache/tez/engine/api/Master.java     |    2 +-
 .../org/apache/tez/engine/records/TezDAGID.java    |  136 ---------------
 .../records/TezDependentTaskCompletionEvent.java   |    1 +
 .../java/org/apache/tez/engine/records/TezID.java  |   95 ----------
 .../tez/engine/records/TezTaskAttemptID.java       |  127 --------------
 .../org/apache/tez/engine/records/TezTaskID.java   |  125 -------------
 .../org/apache/tez/engine/records/TezTaskType.java |   26 ---
 .../org/apache/tez/engine/records/TezVertexID.java |  129 --------------
 tez-engine/pom.xml                                 |    4 +
 .../apache/tez/common/TezEngineTaskContext.java    |    2 +-
 .../org/apache/tez/common/TezTaskReporter.java     |  116 ++++++++++++
 .../tez/common/TezTaskUmbilicalProtocol.java       |   79 +++++++++
 .../engine/common/shuffle/impl/EventFetcher.java   |    2 +-
 .../tez/engine/common/shuffle/impl/Fetcher.java    |    2 +-
 .../engine/common/shuffle/impl/InMemoryReader.java |    2 +-
 .../tez/engine/common/shuffle/impl/MapHost.java    |    2 +-
 .../tez/engine/common/shuffle/impl/MapOutput.java  |    2 +-
 .../engine/common/shuffle/impl/MergeManager.java   |    4 +-
 .../common/shuffle/impl/ShuffleClientMetrics.java  |    2 +-
 .../common/shuffle/impl/ShuffleScheduler.java      |    4 +-
 .../engine/common/sort/impl/ExternalSorter.java    |    2 +-
 .../engine/common/sort/impl/PipelinedSorter.java   |    2 +-
 .../common/sort/impl/dflt/DefaultSorter.java       |    2 +-
 .../task/local/output/TezLocalTaskOutputFiles.java |    2 +-
 .../common/task/local/output/TezTaskOutput.java    |    2 +-
 .../task/local/output/TezTaskOutputFiles.java      |    2 +-
 tez-mapreduce/pom.xml                              |    8 +-
 .../org/apache/hadoop/mapred/LocalJobRunner.java   |    2 +-
 .../apache/tez/mapreduce/combine/MRCombiner.java   |    2 +-
 .../apache/tez/mapreduce/hadoop/IDConverter.java   |    8 +-
 .../apache/tez/mapreduce/hadoop/MRTaskStatus.java  |    2 +-
 .../tez/mapreduce/hadoop/TezTypeConverters.java    |    4 +-
 .../mapreduce/hadoop/mapred/JobContextImpl.java    |    2 +-
 .../hadoop/mapred/TaskAttemptContextImpl.java      |    2 +-
 .../mapreduce/hadoop/mapreduce/JobContextImpl.java |    2 +-
 .../org/apache/tez/mapreduce/processor/MRTask.java |    4 +-
 .../mapreduce/processor/TezTaskReporterImpl.java   |    2 +-
 .../tez/mapreduce/TestUmbilicalProtocol.java       |    2 +-
 .../org/apache/tez/mapreduce/TezTestUtils.java     |    8 +-
 136 files changed, 1090 insertions(+), 1098 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/pom.xml
----------------------------------------------------------------------
diff --git a/tez-common/pom.xml b/tez-common/pom.xml
index eb21ea0..e1ca4c9 100644
--- a/tez-common/pom.xml
+++ b/tez-common/pom.xml
@@ -30,10 +30,6 @@
       <artifactId>hadoop-common</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.apache.tez</groupId>
-      <artifactId>tez-engine-api</artifactId>
-    </dependency>
-    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-api</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/IDUtils.java b/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
index a730e4a..e2cbf35 100644
--- a/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
+++ b/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
@@ -20,11 +20,11 @@ package org.apache.tez.common;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 
 public class IDUtils {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/common/TezTaskContext.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezTaskContext.java b/tez-common/src/main/java/org/apache/tez/common/TezTaskContext.java
index f44c3d5..4eed78b 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezTaskContext.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezTaskContext.java
@@ -23,8 +23,8 @@ import java.io.IOException;
 
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public abstract class TezTaskContext implements Writable {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/common/TezTaskReporter.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezTaskReporter.java b/tez-common/src/main/java/org/apache/tez/common/TezTaskReporter.java
deleted file mode 100644
index 478d833..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/TezTaskReporter.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.common;
-
-import java.io.IOException;
-
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.util.Progressable;
-import org.apache.tez.common.counters.TezCounter;
-import org.apache.tez.engine.api.Master;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
-
-public interface TezTaskReporter extends Progressable, Master {
-
-  public void setStatus(String status);
-
-  public float getProgress();
-
-  public void setProgress(float progress);
-  
-  public void progress();
-
-  public TezCounter getCounter(String group, String name);
-
-  public TezCounter getCounter(Enum<?> name);
-
-  public void incrCounter(String group, String counter, long amount);
-
-  public void incrCounter(Enum<?> key, long amount);
-
-  public void reportFatalError(TezTaskAttemptID taskAttemptId, 
-      Throwable exception, String logMsg);
-
-  public final TezTaskReporter NULL = new TezTaskReporter() {
-
-    @Override
-    public TezTaskDependencyCompletionEventsUpdate getDependentTasksCompletionEvents(
-        int fromEventIdx, int maxEventsToFetch,
-        TezTaskAttemptID reduce) {
-      return null;
-    }
-    
-    @Override
-    public void setStatus(String status) {
-    }
-    
-    @Override
-    public void setProgress(float progress) {
-    }
-    
-    @Override
-    public void progress() {
-    }
-    
-    @Override
-    public void incrCounter(Enum<?> key, long amount) {
-    }
-    
-    @Override
-    public void incrCounter(String group, String counter, long amount) {
-    }
-    
-    @Override
-    public float getProgress() {
-      return 0.0f;
-    }
-    
-    @Override
-    public TezCounter getCounter(Enum<?> name) {
-      return null;
-    }
-    
-    @Override
-    public TezCounter getCounter(String group, String name) {
-      return null;
-    }
-
-    @Override
-    public void reportFatalError(TezTaskAttemptID taskAttemptId,
-        Throwable exception, String logMsg) {
-      // TODO Auto-generated method stub
-      
-    }
-
-    @Override
-    public long getProtocolVersion(String protocol, long clientVersion)
-        throws IOException {
-      // TODO TEZAM3
-      return 0;
-    }
-
-    @Override
-    public ProtocolSignature getProtocolSignature(String protocol,
-        long clientVersion, int clientMethodsHash) throws IOException {
-      // TODO TEZAM3
-      return null;
-    }
-  };
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/common/TezTaskStatus.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezTaskStatus.java b/tez-common/src/main/java/org/apache/tez/common/TezTaskStatus.java
index 0b6d962..de81f87 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezTaskStatus.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezTaskStatus.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.io.Writable;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public interface TezTaskStatus extends Writable {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java b/tez-common/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
deleted file mode 100644
index 8c486aa..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.common;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.tez.common.records.ProceedToCompletionResponse;
-import org.apache.tez.engine.api.Master;
-import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-
-/** Protocol that task child process uses to contact its parent process.  The
- * parent is a daemon which which polls the central master for a new map or
- * reduce task and runs it as a child process.  All communication between child
- * and parent is via this protocol. */ 
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public interface TezTaskUmbilicalProtocol extends Master {
-
-  public static final long versionID = 19L;
-
-  ContainerTask getTask(ContainerContext containerContext) throws IOException;
-  
-  boolean statusUpdate(TezTaskAttemptID taskId, TezTaskStatus taskStatus) 
-  throws IOException, InterruptedException;
-  
-  void reportDiagnosticInfo(TezTaskAttemptID taskid, String trace) throws IOException;
-  
-  boolean ping(TezTaskAttemptID taskid) throws IOException;
-
-  void done(TezTaskAttemptID taskid) throws IOException;
-  
-  void commitPending(TezTaskAttemptID taskId, TezTaskStatus taskStatus) 
-  throws IOException, InterruptedException;  
-
-  boolean canCommit(TezTaskAttemptID taskid) throws IOException;
-
-  void shuffleError(TezTaskAttemptID taskId, String message) throws IOException;
-  
-  void fsError(TezTaskAttemptID taskId, String message) throws IOException;
-
-  void fatalError(TezTaskAttemptID taskId, String message) throws IOException;
-  
-  // TODO TEZAM5 Can commitPending and outputReady be collapsed into a single
-  // call.
-  // IAC outputReady followed by commit is a little confusing - since the output
-  // isn't really in place till a commit is called. Maybe rename to
-  // processingComplete or some such.
-  
-  // TODO EVENTUALLY This is not the most useful API. Once there's some kind of
-  // support for the Task handing output over to the Container, this won't rally
-  // be required. i.e. InMemShuffle running as a service in the Container, or
-  // the second task in getTask(). ContainerUmbilical would include getTask and
-  // getServices...
-  
-  void outputReady(TezTaskAttemptID taskAttemptId, OutputContext outputContext)
-      throws IOException;
-  
-  ProceedToCompletionResponse
-      proceedToCompletion(TezTaskAttemptID taskAttemptId) throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
new file mode 100644
index 0000000..e977dc0
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.text.NumberFormat;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+
+
+/**
+ * TezDAGID represents the immutable and unique identifier for
+ * a Tez DAG.
+ *
+ * TezDAGID consists of 2 parts. The first part is the {@link ApplicationId},
+ * that is the YARN Application ID that this DAG belongs to. The second part is
+ * the DAG number.
+ *
+ * @see ApplicationId
+ */
+public class TezDAGID extends TezID {
+
+  public static final String DAG = "dag";
+  protected static final NumberFormat idFormat = NumberFormat.getInstance();
+  static {
+    idFormat.setGroupingUsed(false);
+    idFormat.setMinimumIntegerDigits(6);
+  }
+
+  private ApplicationId applicationId;
+
+  public TezDAGID() {
+  }
+
+  /**
+   * Constructs a DAGID object from given {@link ApplicationId}.
+   * @param applicationId Application that this dag belongs to
+   * @param id the dag number
+   */
+  public TezDAGID(ApplicationId applicationId, int id) {
+    super(id);
+    if(applicationId == null) {
+      throw new IllegalArgumentException("applicationId cannot be null");
+    }
+    this.applicationId = applicationId;
+  }
+
+  /**
+   * Constructs a DAGID object from given parts.
+   * @param yarnRMIdentifier YARN RM identifier
+   * @param applicationId application number
+   * @param id the dag number
+   */
+  public TezDAGID(String yarnRMIdentifier, int appId, int id) {
+    this(BuilderUtils.newApplicationId(Long.valueOf(yarnRMIdentifier),
+        appId), id);
+  }
+
+  /** Returns the {@link ApplicationId} object that this dag belongs to */
+  public ApplicationId getApplicationId() {
+    return applicationId;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!super.equals(o))
+      return false;
+
+    TezDAGID that = (TezDAGID)o;
+    return this.applicationId.equals(that.applicationId);
+  }
+
+  /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
+  @Override
+  public int compareTo(TezID o) {
+    TezDAGID that = (TezDAGID)o;
+    return this.applicationId.compareTo(that.applicationId);
+  }
+
+  @Override
+  public String toString() {
+    return appendTo(new StringBuilder(DAG)).toString();
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    applicationId = BuilderUtils.newApplicationId(in.readLong(), in.readInt());
+    super.readFields(in);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeLong(applicationId.getClusterTimestamp());
+    out.writeInt(applicationId.getId());
+    super.write(out);
+  }
+
+  /**
+   * Add the unique string to the given builder.
+   * @param builder the builder to append to
+   * @return the builder that was passed in
+   */
+  protected StringBuilder appendTo(StringBuilder builder) {
+    return builder.append(SEPARATOR).
+                 append(applicationId.getClusterTimestamp()).
+                 append(SEPARATOR).
+                 append(applicationId.getId()).
+                 append(SEPARATOR).
+                 append(idFormat.format(id));
+  }
+
+  @Override
+  public int hashCode() {
+    return applicationId.hashCode() * 524287 + id;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java
new file mode 100644
index 0000000..7fe5d8c
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+
+/**
+ * A general identifier, which internally stores the id
+ * as an integer. This is the super class of {@link TezJobID}, 
+ * {@link TezTaskID} and {@link TezTaskAttemptID}.
+ * 
+ * @see TezTaskID
+ * @see TezTaskAttemptID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public abstract class TezID implements WritableComparable<TezID> {
+  public static final char SEPARATOR = '_';
+  protected int id;
+
+  /** constructs an ID object from the given int */
+  public TezID(int id) {
+    this.id = id;
+  }
+
+  protected TezID() {
+  }
+
+  /** returns the int which represents the identifier */
+  public int getId() {
+    return id;
+  }
+
+  @Override
+  public String toString() {
+    return String.valueOf(id);
+  }
+
+  @Override
+  public int hashCode() {
+    return id;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o)
+      return true;
+    if(o == null)
+      return false;
+    if (o.getClass() == this.getClass()) {
+      TezID that = (TezID) o;
+      return this.id == that.id;
+    }
+    else
+      return false;
+  }
+
+  /** Compare IDs by associated numbers */
+  public int compareTo(TezID that) {
+    return this.id - that.id;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    this.id = in.readInt();
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeInt(id);
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java
new file mode 100644
index 0000000..4b2f424
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.tez.dag.records.TezTaskID;
+
+/**
+ * TezTaskAttemptID represents the immutable and unique identifier for
+ * a task attempt. Each task attempt is one particular instance of a Tez Task
+ * identified by its TezTaskID.
+ *
+ * TezTaskAttemptID consists of 2 parts. First part is the
+ * {@link TezTaskID}, that this TaskAttemptID belongs to.
+ * Second part is the task attempt number. <br>
+ * <p>
+ * Applications should never construct or parse TaskAttemptID strings
+ * , but rather use appropriate constructors or {@link #forName(String)}
+ * method.
+ *
+ * @see TezTaskID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezTaskAttemptID extends TezID {
+  public static final String ATTEMPT = "attempt";
+  private TezTaskID taskId;
+  
+  public TezTaskAttemptID() {
+    taskId = new TezTaskID();
+  }
+  
+  /**
+   * Constructs a TaskAttemptID object from given {@link TezTaskID}.  
+   * @param taskId TaskID that this task belongs to  
+   * @param id the task attempt number
+   */
+  public TezTaskAttemptID(TezTaskID taskId, int id) {
+    super(id);
+    if(taskId == null) {
+      throw new IllegalArgumentException("taskId cannot be null");
+    }
+    this.taskId = taskId;
+  }
+
+  /** Returns the {@link TezTaskID} object that this task attempt belongs to */
+  public TezTaskID getTaskID() {
+    return taskId;
+  }
+  
+  @Override
+  public boolean equals(Object o) {
+    if (!super.equals(o))
+      return false;
+
+    TezTaskAttemptID that = (TezTaskAttemptID)o;
+    return this.taskId.equals(that.taskId);
+  }
+  
+  /**
+   * Add the unique string to the StringBuilder
+   * @param builder the builder to append ot
+   * @return the builder that was passed in.
+   */
+  protected StringBuilder appendTo(StringBuilder builder) {
+    return taskId.appendTo(builder).append(SEPARATOR).append(id);
+  }
+  
+  @Override
+  public int hashCode() {
+    return taskId.hashCode() * 539501 + id;
+  }
+  
+  /**Compare TaskIds by first tipIds, then by task numbers. */
+  @Override
+  public int compareTo(TezID o) {
+    TezTaskAttemptID that = (TezTaskAttemptID)o;
+    int tipComp = this.taskId.compareTo(that.taskId);
+    if(tipComp == 0) {
+      return this.id - that.id;
+    }
+    else return tipComp;
+  }
+  @Override
+  public String toString() { 
+    return appendTo(new StringBuilder(ATTEMPT)).toString();
+  }
+  
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    taskId.readFields(in);
+    super.readFields(in);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    taskId.write(out);
+    super.write(out);
+  }
+
+  // FIXME TEZ DAG needs to be removed
+  public static TezTaskAttemptID read(DataInput in) throws IOException {
+    TezTaskAttemptID tId = new TezTaskAttemptID();
+    tId.readFields(in);
+    return tId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
new file mode 100644
index 0000000..d52e83b
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.tez.dag.records.TezVertexID;
+
+
+/**
+ * TaskID represents the immutable and unique identifier for 
+ * a Tez Task. Each TaskID encompasses multiple attempts made to
+ * execute the Tez Task, each of which are uniquely identified by
+ * their TezTaskAttemptID.
+ * 
+ * @see TezTaskAttemptID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezTaskID extends TezID {
+  public static final String TASK = "task";
+  protected static final NumberFormat idFormat = NumberFormat.getInstance();
+  static {
+    idFormat.setGroupingUsed(false);
+    idFormat.setMinimumIntegerDigits(6);
+  }
+  
+  private TezVertexID vertexId;
+  
+  public TezTaskID() {
+    vertexId = new TezVertexID();
+  }
+  
+  /**
+   * Constructs a TaskID object from given {@link MRxApplicationID}.  
+   * @param jobId JobID that this tip belongs to 
+   * @param type the {@link TezTaskType} of the task 
+   * @param id the tip number
+   */
+  public TezTaskID(TezVertexID vertexId, int id) {
+    super(id);
+    if(vertexId == null) {
+      throw new IllegalArgumentException("vertexId cannot be null");
+    }
+    this.vertexId = vertexId;
+  }
+  
+  /** Returns the {@link TezVertexID} object that this task belongs to */
+  public TezVertexID getVertexID() {
+    return vertexId;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!super.equals(o))
+      return false;
+
+    TezTaskID that = (TezTaskID)o;
+    return this.vertexId.equals(that.vertexId);
+  }
+
+  /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
+  @Override
+  public int compareTo(TezID o) {
+    TezTaskID that = (TezTaskID)o;
+    int vertexComp = this.vertexId.compareTo(that.vertexId);
+    if(vertexComp == 0) {
+      return this.id - that.id;
+    }
+    else return vertexComp;
+  }
+  @Override
+  public String toString() { 
+    return appendTo(new StringBuilder(TASK)).toString();
+  }
+
+  /**
+   * Add the unique string to the given builder.
+   * @param builder the builder to append to
+   * @return the builder that was passed in
+   */
+  protected StringBuilder appendTo(StringBuilder builder) {
+    return vertexId.appendTo(builder).
+                 append(SEPARATOR).
+                 append(idFormat.format(id));
+  }
+  
+  @Override
+  public int hashCode() {
+    return vertexId.hashCode() * 535013 + id;
+  }
+  
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    vertexId.readFields(in);
+    super.readFields(in);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    vertexId.write(out);
+    super.write(out);
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
new file mode 100644
index 0000000..8b2a272
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * TezVertexID represents the immutable and unique identifier for
+ * a Vertex in a Tez DAG. Each TezVertexID encompasses multiple Tez Tasks.
+ *
+ * TezVertezID consists of 2 parts. The first part is the {@link TezDAGID},
+ * that is the Tez DAG that this vertex belongs to. The second part is
+ * the vertex number.
+ *
+ * @see TezDAGID
+ * @see TezTaskID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezVertexID extends TezID {
+  public static final String VERTEX = "vertex";
+  protected static final NumberFormat idFormat = NumberFormat.getInstance();
+  static {
+    idFormat.setGroupingUsed(false);
+    idFormat.setMinimumIntegerDigits(6);
+  }
+  
+  private TezDAGID dagId;
+  
+  public TezVertexID() {
+  }
+  
+  /**
+   * Constructs a TaskID object from given {@link TezDAGID}.
+   * @param applicationId JobID that this tip belongs to
+   * @param type the {@link TezTaskType} of the task
+   * @param id the tip number
+   */
+  public TezVertexID(TezDAGID dagId, int id) {
+    super(id);
+    if(dagId == null) {
+      throw new IllegalArgumentException("dagId cannot be null");
+    }
+    this.dagId = dagId;
+  }
+  
+  /** Returns the {@link TezDAGID} object that this tip belongs to */
+  public TezDAGID getDAGId() {
+    return dagId;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!super.equals(o))
+      return false;
+
+    TezVertexID that = (TezVertexID)o;
+    return this.dagId.equals(that.dagId);
+  }
+
+  /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
+  @Override
+  public int compareTo(TezID o) {
+    TezVertexID that = (TezVertexID)o;
+    return this.dagId.compareTo(that.dagId);
+  }
+  
+  @Override
+  public String toString() { 
+    return appendTo(new StringBuilder(VERTEX)).toString();
+  }
+  
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    dagId = new TezDAGID();
+    dagId.readFields(in);
+    super.readFields(in);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    dagId.write(out);
+    super.write(out);
+  }
+
+  /**
+   * Add the unique string to the given builder.
+   * @param builder the builder to append to
+   * @return the builder that was passed in
+   */
+  protected StringBuilder appendTo(StringBuilder builder) {
+    return builder.append(SEPARATOR).
+        append(dagId.getApplicationId().getClusterTimestamp()).
+        append(SEPARATOR).
+        append(dagId.getApplicationId().getId()).
+        append(SEPARATOR).
+        append(dagId.getId()).
+        append(SEPARATOR).
+        append(idFormat.format(id));
+  }
+  
+  @Override
+  public int hashCode() {
+    return dagId.hashCode() * 530017 + id;
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
deleted file mode 100644
index e57b529..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.client;
-
-public class DAGStatus {
-  public static enum State {
-    RUNNING(1),
-    SUCCEEDED(2),
-    FAILED(3),
-    PREP(4),
-    KILLED(5);
-    
-    int value;
-    
-    State(int value) {
-      this.value = value;
-    }
-    
-    public int getValue() {
-      return value; 
-    }
-    
-  };
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
deleted file mode 100644
index 34f3a5c..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.client;
-
-public class VertexStatus {
-
-  public static enum State {
-    RUNNING(1),
-    SUCCEEDED(2),
-    FAILED(3),
-    PREP(4),
-    KILLED(5);
-
-    int value;
-
-    State(int value) {
-      this.value = value;
-    }
-
-    public int getValue() {
-      return value;
-    }
-
-  };
-
-  private State runState;
-
-  public synchronized State getState() {
-    return runState;
-  }
-
-  public synchronized void setState(State state) {
-    this.runState = state;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index 8390e5f..fc03bea 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -31,6 +31,10 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.tez</groupId>
+      <artifactId>tez-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
       <artifactId>tez-mapreduce</artifactId>
     </dependency>
     <dependency>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
index bb1016a..a3b72c7 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
@@ -33,12 +33,12 @@ import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.YarnException;
-import org.apache.tez.dag.api.client.VertexStatus;
 import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
 import org.apache.tez.dag.api.impl.VertexContext;
 import org.apache.tez.dag.api.impl.VertexOutputCommitter;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.api.impl.VertexStatus;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 public class MRVertexOutputCommitter extends VertexOutputCommitter {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/hadoop/mapred/TezMRTypeConverter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/TezMRTypeConverter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/TezMRTypeConverter.java
index 4780581..a47ef40 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/TezMRTypeConverter.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/TezMRTypeConverter.java
@@ -20,10 +20,10 @@ package org.apache.hadoop.mapred;
 
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 
 public class TezMRTypeConverter {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 1854002..f140c3a 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -70,8 +70,8 @@ import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.TezEngineTaskContext;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.api.Task;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 import org.apache.tez.engine.runtime.RuntimeUtils;
 import org.apache.tez.mapreduce.hadoop.DeprecatedKeys;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
index 1e5ffa5..5f36a58 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
@@ -22,10 +22,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.tez.dag.api.records.AMInfo;
 import org.apache.tez.dag.app.dag.DAGReport;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 
 public class TezBuilderUtils {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
new file mode 100644
index 0000000..6ede497
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
@@ -0,0 +1,39 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.impl;
+
+public class DAGStatus {
+  public static enum State {
+    RUNNING(1),
+    SUCCEEDED(2),
+    FAILED(3),
+    PREP(4),
+    KILLED(5);
+    
+    int value;
+    
+    State(int value) {
+      this.value = value;
+    }
+    
+    public int getValue() {
+      return value; 
+    }
+    
+  };
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
index 77995a4..102bdec 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.api.impl;
 
 import java.io.IOException;
 
-import org.apache.tez.dag.api.client.VertexStatus.State;
+import org.apache.tez.dag.api.impl.VertexStatus.State;
 
 public class NullVertexOutputCommitter extends VertexOutputCommitter {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
index 9789b5d..181aea5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
@@ -20,8 +20,8 @@ package org.apache.tez.dag.api.impl;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
 
 public interface VertexContext {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
index 3024d6e..fe3e2d5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
@@ -22,7 +22,6 @@ import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.tez.dag.api.client.VertexStatus;
 
 @Public
 @Unstable

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
new file mode 100644
index 0000000..9ba2ece
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.impl;
+
+public class VertexStatus {
+
+  public static enum State {
+    RUNNING(1),
+    SUCCEEDED(2),
+    FAILED(3),
+    PREP(4),
+    KILLED(5);
+
+    int value;
+
+    State(int value) {
+      this.value = value;
+    }
+
+    public int getValue() {
+      return value;
+    }
+
+  };
+
+  private State runState;
+
+  public synchronized State getState() {
+    return runState;
+  }
+
+  public synchronized void setState(State state) {
+    this.runState = state;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
index 044afbb..a5a9e73 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.api.records;
 
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public interface TaskAttemptReport {
   public abstract TezTaskAttemptID getTaskAttemptId();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
index a6605ec..a0834d7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
@@ -21,8 +21,8 @@ package org.apache.tez.dag.api.records;
 import java.util.List;
 
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
 
 public interface TaskReport {
   public abstract TezTaskID getTaskId();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
index df96d67..041de56 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.rm.container.AMContainerMap;
 import org.apache.tez.dag.app.rm.node.AMNodeMap;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
 
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 174d210..a0c2676 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -101,8 +101,8 @@ import org.apache.tez.dag.history.DAGHistoryEvent;
 import org.apache.tez.dag.history.HistoryEventHandler;
 import org.apache.tez.dag.history.avro.HistoryEventType;
 import org.apache.tez.dag.history.events.AMStartedEvent;
+import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.engine.common.security.JobTokenSecretManager;
-import org.apache.tez.engine.records.TezDAGID;
 
 /**
  * The Map-Reduce Application Master.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
index ef29d06..e80c8b3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.app;
 import java.net.InetSocketAddress;
 
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 /**
  * This class listens for changes to the state of a Task.
  */

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index c1c88e2..d974cd8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -55,10 +55,10 @@ import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttempt
 import org.apache.tez.dag.app.rm.container.AMContainerImpl;
 import org.apache.tez.dag.app.rm.container.AMContainerTask;
 import org.apache.tez.dag.app.security.authorize.MRAMPolicyProvider;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.common.security.JobTokenSecretManager;
 import org.apache.tez.engine.records.OutputContext;
 import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
 
 @SuppressWarnings("unchecked")

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
index 198ec7f..7239484 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEvent;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventDiagnosticsUpdate;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index da0e38a..f2c3791 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -26,8 +26,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.api.VertexLocationHint;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
 
 /**
  * Main interface to interact with the job. Provides only getters. 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
index 17f1243..6cfc2c2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
@@ -23,8 +23,8 @@ import java.util.Map;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.records.TaskReport;
 import org.apache.tez.dag.api.records.TaskState;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
 
 /**
  * Read only view of Task.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
index 0b8779f..d4e038a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.records.TaskAttemptReport;
 import org.apache.tez.dag.api.records.TaskAttemptState;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 /**
  * Read only view of TaskAttempt.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index 1e253aa..ace8a8b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -29,9 +29,9 @@ import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
 
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexScheduler.java
index 05a30c3..df9c28e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexScheduler.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.app.dag;
 
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public interface VertexScheduler {
   void onVertexStarted();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java
index 35cd236..1ec0222 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java
@@ -19,7 +19,7 @@
 package org.apache.tez.dag.app.dag.event;
 
 import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
 
 /**
  * This class encapsulates job related events.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java
index aba2b9b..da0724d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.app.dag.event;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
 
 public class DAGEventCounterUpdate extends DAGEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java
index a2ac101..bde94a0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.app.dag.event;
 
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
 
 public class DAGEventDiagnosticsUpdate extends DAGEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java
index 4d6cf91..80c7f42 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java
@@ -19,7 +19,7 @@
 package org.apache.tez.dag.app.dag.event;
 
 import org.apache.tez.dag.app.dag.VertexState;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezVertexID;
 
 public class DAGEventVertexCompleted extends DAGEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGFinishEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGFinishEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGFinishEvent.java
index 15f91e3..817909f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGFinishEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGFinishEvent.java
@@ -19,7 +19,7 @@
 package org.apache.tez.dag.app.dag.event;
 
 import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
 
 public class DAGFinishEvent extends AbstractEvent<DAGFinishEvent.Type> {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java
index 7bbb6b7..56c03e3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java
@@ -19,7 +19,7 @@
 package org.apache.tez.dag.app.dag.event;
 
 import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 /**
  * This class encapsulates task attempt related events.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java
index c2599f3..87aa313 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java
@@ -17,7 +17,7 @@
 
 package org.apache.tez.dag.app.dag.event;
 
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskAttemptEventContainerTerminated extends TaskAttemptEvent
     implements DiagnosableEvent {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
index bc82387..8c88cb9 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
@@ -17,7 +17,7 @@
 
 package org.apache.tez.dag.app.dag.event;
 
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskAttemptEventContainerTerminating extends TaskAttemptEvent
     implements DiagnosableEvent {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventDiagnosticsUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventDiagnosticsUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventDiagnosticsUpdate.java
index 748530f..0a1ca74 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventDiagnosticsUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventDiagnosticsUpdate.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.app.dag.event;
 
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskAttemptEventDiagnosticsUpdate extends TaskAttemptEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
index be23e17..bba8027 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.app.dag.event;
 
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskAttemptEventFailRequest extends TaskAttemptEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java
index b0bc108..9bceb1d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java
@@ -17,7 +17,7 @@
 */
 package org.apache.tez.dag.app.dag.event;
 
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskAttemptEventKillRequest extends TaskAttemptEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
index 316f2ce..4fc718d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
@@ -17,7 +17,7 @@
 
 package org.apache.tez.dag.app.dag.event;
 
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskAttemptEventNodeFailed extends TaskAttemptEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputConsumable.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputConsumable.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputConsumable.java
index ec653d4..f10209f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputConsumable.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputConsumable.java
@@ -17,8 +17,8 @@
 
 package org.apache.tez.dag.app.dag.event;
 
+import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
 
 public class TaskAttemptEventOutputConsumable extends TaskAttemptEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java
index 2f99faf..a7d4c9a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java
@@ -18,7 +18,7 @@
 package org.apache.tez.dag.app.dag.event;
 
 import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java
index 27897af..5a13fc1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java
@@ -22,7 +22,7 @@ import java.util.Map;
 
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskAttemptEventStartedRemotely extends TaskAttemptEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
index 50007fa..ef23638 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
@@ -22,7 +22,7 @@ import java.util.List;
 
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.records.TaskAttemptState;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskAttemptEventStatusUpdate extends TaskAttemptEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java
index c3c562b..c7e5faa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java
@@ -19,7 +19,7 @@
 package org.apache.tez.dag.app.dag.event;
 
 import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
 
 /**
  * this class encapsulates task related events.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java
index 66aff4b..59c7363 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.app.dag.event;
 
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class TaskEventTAUpdate extends TaskEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java
index 4cfbfea..9e94eb5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java
@@ -19,7 +19,7 @@
 package org.apache.tez.dag.app.dag.event;
 
 import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezVertexID;
 
 /**
  * this class encapsulates vertex related events.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java
index 209bb8b..27d9a02 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java
@@ -18,8 +18,8 @@
 
 package org.apache.tez.dag.app.dag.event;
 
+import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezVertexID;
 
 public class VertexEventSourceTaskAttemptCompleted extends VertexEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java
index 13bd89c..cc97aa2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.app.dag.event;
 
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezVertexID;
 
 public class VertexEventSourceVertexStarted extends VertexEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptFetchFailure.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptFetchFailure.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptFetchFailure.java
index 723338b..5b2b955 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptFetchFailure.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptFetchFailure.java
@@ -19,7 +19,7 @@ package org.apache.tez.dag.app.dag.event;
 
 import java.util.List;
 
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
 
 public class VertexEventTaskAttemptFetchFailure extends VertexEvent {
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
index 39b0f92..79512c7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
@@ -19,7 +19,7 @@
 package org.apache.tez.dag.app.dag.event;
 
 import org.apache.tez.dag.api.records.TaskState;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
 
 
 public class VertexEventTaskCompleted extends VertexEvent {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java
index 5fc09d9..9706246 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java
@@ -17,7 +17,7 @@
 */
 package org.apache.tez.dag.app.dag.event;
 
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
 
 public class VertexEventTaskReschedule extends VertexEvent {
   private TezTaskID taskID;


[02/50] [abbrv] TEZ-25: Change DAG representation from Configuration object to structured protobuf message

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/test/java/org/apache/tez/dag/api/TestVertexLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/api/TestVertexLocationHint.java b/tez-dag/src/test/java/org/apache/tez/dag/api/TestVertexLocationHint.java
deleted file mode 100644
index eb8c781..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/api/TestVertexLocationHint.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestVertexLocationHint {
-
-  private DataInput in;
-  private DataOutput out;
-  private ByteArrayOutputStream bOut;
-
-  @Before
-  public void setup() {
-    bOut = new ByteArrayOutputStream();
-    out = new DataOutputStream(bOut);
-  }
-
-  @After
-  public void teardown() {
-    in = null;
-    out = null;
-    bOut = null;
-  }
-
-  @Test
-  public void testNullTaskLocationHintSerDes() throws IOException {
-    TaskLocationHint expected = new TaskLocationHint(null, null);
-    expected.write(out);
-    in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
-    TaskLocationHint actual = new TaskLocationHint();
-    actual.readFields(in);
-    Assert.assertNull(actual.getDataLocalHosts());
-    Assert.assertNull(actual.getRacks());
-  }
-
-  @Test
-  public void testTaskLocationHintSerDes() throws IOException {
-    String[] hosts = { "h1", "h2", "", null };
-    String[] racks = { "r1", "r2" };
-    TaskLocationHint expected = new TaskLocationHint(hosts, racks);
-    expected.write(out);
-    in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
-    TaskLocationHint actual = new TaskLocationHint();
-    actual.readFields(in);
-    Assert.assertNotNull(actual.getDataLocalHosts());
-    Assert.assertNotNull(actual.getRacks());
-    Assert.assertArrayEquals(hosts, actual.getDataLocalHosts());
-    Assert.assertArrayEquals(racks, actual.getRacks());
-  }
-
-  @Test
-  public void testTaskLocationHintSerDes2() throws IOException {
-    String[] hosts = null;
-    String[] racks = { "r1", "r2" };
-    TaskLocationHint expected = new TaskLocationHint(hosts, racks);
-    expected.write(out);
-    in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
-    TaskLocationHint actual = new TaskLocationHint();
-    actual.readFields(in);
-    Assert.assertNull(actual.getDataLocalHosts());
-    Assert.assertNotNull(actual.getRacks());
-    Assert.assertArrayEquals(racks, actual.getRacks());
-  }
-
-  @Test
-  public void testEmptyVertexLocationHintSerDes() throws IOException {
-    VertexLocationHint expected = new VertexLocationHint(0);
-    expected.write(out);
-    in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
-    VertexLocationHint actual = new VertexLocationHint();
-    actual.readFields(in);
-    Assert.assertEquals(0, actual.getNumTasks());
-    Assert.assertNotNull(actual.getTaskLocationHints());
-    Assert.assertEquals(0, actual.getTaskLocationHints().length);
-  }
-
-  @Test
-  public void testVertexLocationHintSerDes() throws IOException {
-    String[] hosts = { "h1", "h2", "", null };
-    String[] racks = { "r1", "r2" };
-    VertexLocationHint expected = new VertexLocationHint(4);
-    expected.getTaskLocationHints()[0] = new TaskLocationHint(hosts, racks);
-    expected.getTaskLocationHints()[1] = null;
-    expected.getTaskLocationHints()[2] = new TaskLocationHint(null, racks);
-    expected.getTaskLocationHints()[3] = new TaskLocationHint(hosts, null);
-    expected.write(out);
-    in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
-    VertexLocationHint actual = new VertexLocationHint();
-    actual.readFields(in);
-
-    Assert.assertEquals(4, actual.getNumTasks());
-    Assert.assertNotNull(actual.getTaskLocationHints());
-    Assert.assertEquals(4, actual.getTaskLocationHints().length);
-
-    Assert.assertNotNull(actual.getTaskLocationHints()[0]);
-    Assert.assertNull(actual.getTaskLocationHints()[1]);
-    Assert.assertNotNull(actual.getTaskLocationHints()[2]);
-    Assert.assertNotNull(actual.getTaskLocationHints()[3]);
-
-    Assert.assertArrayEquals(
-        expected.getTaskLocationHints()[0].getDataLocalHosts(),
-        actual.getTaskLocationHints()[0].getDataLocalHosts());
-    Assert.assertArrayEquals(
-        expected.getTaskLocationHints()[0].getRacks(),
-        actual.getTaskLocationHints()[0].getRacks());
-    Assert.assertNull(
-        actual.getTaskLocationHints()[2].getDataLocalHosts());
-    Assert.assertArrayEquals(
-        expected.getTaskLocationHints()[2].getRacks(),
-        actual.getTaskLocationHints()[2].getRacks());
-    Assert.assertArrayEquals(
-        expected.getTaskLocationHints()[3].getDataLocalHosts(),
-        actual.getTaskLocationHints()[3].getDataLocalHosts());
-    Assert.assertNull(
-        actual.getTaskLocationHints()[3].getRacks());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 715a364..9042846 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -99,7 +99,7 @@ import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DAGConfiguration;
+import org.apache.tez.dag.api.DAGPlan.JobPlan;
 import org.apache.tez.dag.api.Edge;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;
@@ -737,6 +737,14 @@ public class YARNRunner implements ClientProtocol {
     List<String> vargs = new ArrayList<String>(8);
     vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
 
+//[Debug AppMaster] Current simplest way to attach debugger to AppMaster
+// Uncomment the following, then launch a regular job, eg 
+// >hadoop jar {path}\hadoop-mapreduce-examples-3.0.0-SNAPSHOT.jar pi 2 2                  
+//     LOG.error(" !!!!!!!!!");
+//     LOG.error(" !!!!!!!!! Launching AppMaster in debug/suspend mode.  Attach to port 8002");
+//     LOG.error(" !!!!!!!!!");
+//     vargs.add("-Xdebug -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=8002,server=y,suspend=y");
+    
     // FIXME set up logging related properties
     // TODO -Dtez.root.logger??
     // MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
@@ -795,24 +803,38 @@ public class YARNRunner implements ClientProtocol {
 
     setDAGParamsFromMRConf(dag);
 
-    // FIXME add serialized dag conf
-    DAGConfiguration dagConf = dag.serializeDag();
-    
-    Path dagConfFilePath = new Path(jobSubmitDir,
-        TezConfiguration.DAG_AM_PLAN_CONFIG_XML);
-    FSDataOutputStream dagConfOut =
-        FileSystem.create(fs, dagConfFilePath,
-            new FsPermission(DAG_FILE_PERMISSION));
+    // emit protobuf DAG file style
+    JobPlan dagPB = dag.createDag();
+    FSDataOutputStream dagPBOutBinaryStream = null;
+    FSDataOutputStream dagPBOutTextStream = null;
+    Path binaryPath =  new Path(jobSubmitDir, TezConfiguration.DAG_AM_PLAN_PB_BINARY);
+    Path textPath =  new Path(jobSubmitDir, TezConfiguration.DAG_AM_PLAN_PB_TEXT);
     try {
-      dagConf.writeXml(dagConfOut);
+      //binary output
+      dagPBOutBinaryStream = FileSystem.create(fs, binaryPath,
+          new FsPermission(DAG_FILE_PERMISSION));
+      dagPB.writeTo(dagPBOutBinaryStream);
+
+      // text / human-readable output
+      dagPBOutTextStream = FileSystem.create(fs, textPath,
+          new FsPermission(DAG_FILE_PERMISSION));
+      dagPBOutTextStream.writeUTF(dagPB.toString());
     } finally {
-      dagConfOut.close();
+      if(dagPBOutBinaryStream != null){
+        dagPBOutBinaryStream.close();
+      }
+      if(dagPBOutTextStream != null){
+        dagPBOutTextStream.close();
+      }
     }
-    localResources.put(TezConfiguration.DAG_AM_PLAN_CONFIG_XML,
+
+    localResources.put(TezConfiguration.DAG_AM_PLAN_PB_BINARY,
         createApplicationResource(defaultFileContext,
-            dagConfFilePath, LocalResourceType.FILE));
+            binaryPath, LocalResourceType.FILE));
 
-    // FIXME add tez conf if needed
+    localResources.put(TezConfiguration.DAG_AM_PLAN_PB_TEXT,
+        createApplicationResource(defaultFileContext,
+            textPath, LocalResourceType.FILE));
 
     // FIXME are we using MR acls for tez?
     Map<ApplicationAccessType, String> acls


[12/50] [abbrv] git commit: TEZ-109. Create tests for VertexImpl.

Posted by ss...@apache.org.
TEZ-109. Create tests for VertexImpl.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/38b410e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/38b410e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/38b410e3

Branch: refs/heads/master
Commit: 38b410e3116919d31fbb9bab2ba4cc7c21cc9fee
Parents: 645c470
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu May 16 12:35:12 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu May 16 12:35:12 2013 -0700

----------------------------------------------------------------------
 pom.xml                                            |    1 +
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |   70 +-
 .../tez/dag/app/dag/impl/TestVertexImpl.java       |  950 +++++++++++++++
 3 files changed, 1000 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/38b410e3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5c545c5..7fa336a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -294,6 +294,7 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-surefire-plugin</artifactId>
+          <version>2.14.1</version>
           <configuration>
             <forkMode>always</forkMode>
             <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/38b410e3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 3f3ac49..aedb386 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -101,6 +101,8 @@ import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
 import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
 
+import com.google.common.annotations.VisibleForTesting;
+
 
 /** Implementation of Vertex interface. Maintains the state machines of Vertex.
  * The read and write calls use ReadWriteLock for concurrency.
@@ -112,7 +114,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   private static final String LINE_SEPARATOR = System
       .getProperty("line.separator");
   private static final TezDependentTaskCompletionEvent[]
-    EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS = new TezDependentTaskCompletionEvent[0];
+      EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS =
+      new TezDependentTaskCompletionEvent[0];
 
   private static final Log LOG = LogFactory.getLog(VertexImpl.class);
 
@@ -154,7 +157,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   private final List<String> diagnostics = new ArrayList<String>();
 
   //task/attempt related datastructures
-  private final Map<TezTaskID, Integer> successSourceAttemptCompletionEventNoMap =
+  @VisibleForTesting
+  final Map<TezTaskID, Integer> successSourceAttemptCompletionEventNoMap =
     new HashMap<TezTaskID, Integer>();
   private final Map<TezTaskAttemptID, Integer> fetchFailuresMapping =
     new HashMap<TezTaskAttemptID, Integer>();
@@ -214,12 +218,14 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
               SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
           .addTransition
               (VertexState.RUNNING,
-              EnumSet.of(VertexState.RUNNING, VertexState.SUCCEEDED, VertexState.FAILED),
+              EnumSet.of(VertexState.RUNNING, VertexState.KILLED,
+                  VertexState.SUCCEEDED, VertexState.FAILED),
               VertexEventType.V_TASK_COMPLETED,
               new TaskCompletedTransition())
           .addTransition
               (VertexState.RUNNING,
-              EnumSet.of(VertexState.RUNNING, VertexState.SUCCEEDED, VertexState.FAILED),
+              EnumSet.of(VertexState.RUNNING, VertexState.SUCCEEDED,
+                  VertexState.FAILED),
               VertexEventType.V_COMPLETED,
               new VertexNoTasksCompletedTransition())
           .addTransition(VertexState.RUNNING, VertexState.KILL_WAIT,
@@ -308,7 +314,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
           // create the topology tables
           .installTopology();
 
-  private final StateMachine<VertexState, VertexEventType, VertexEvent> stateMachine;
+  private final StateMachine<VertexState, VertexEventType, VertexEvent>
+      stateMachine;
 
   //changing fields while the vertex is running
   private int numTasks;
@@ -341,10 +348,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   private VertexLocationHint vertexLocationHint;
   private Map<String, LocalResource> localResources;
   private Map<String, String> environment;
-  private String javaOpts;
+  private final String javaOpts;
   
-  public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, String vertexName,
-      TezConfiguration conf, EventHandler eventHandler,
+  public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, 
+      String vertexName, TezConfiguration conf, EventHandler eventHandler,
       TaskAttemptListener taskAttemptListener,
       Token<JobTokenIdentifier> jobToken,
       Credentials fsTokenCredentials, Clock clock,
@@ -573,9 +580,11 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       try {
          getStateMachine().doTransition(event.getType(), event);
       } catch (InvalidStateTransitonException e) {
-        LOG.error("Can't handle this event at current state", e);
-        addDiagnostic("Invalid event " + event.getType() +
-            " on Job " + this.vertexId);
+        String message = "Invalid event " + event.getType() +
+            " on vertex " + this.vertexId +
+            " at current state " + oldState;
+        LOG.error("Can't handle " + message, e);
+        addDiagnostic(message);
         eventHandler.handle(new VertexEvent(this.vertexId,
             VertexEventType.INTERNAL_ERROR));
       }
@@ -957,6 +966,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     @Override
     public void transition(VertexImpl vertex, VertexEvent event) {
       vertex.setFinishTime();
+      vertex.addDiagnostic("Vertex received Kill in NEW state.");
       vertex.finished(VertexState.KILLED);
     }
   }
@@ -966,7 +976,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     @Override
     public void transition(VertexImpl vertex, VertexEvent event) {
       vertex.abortVertex(VertexStatus.State.KILLED);
-      vertex.addDiagnostic("Job received Kill in INITED state.");
+      vertex.addDiagnostic("Vertex received Kill in INITED state.");
       vertex.finished(VertexState.KILLED);
     }
   }
@@ -975,7 +985,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       implements SingleArcTransition<VertexImpl, VertexEvent> {
     @Override
     public void transition(VertexImpl vertex, VertexEvent event) {
-      vertex.addDiagnostic("Job received Kill while in RUNNING state.");
+      vertex.addDiagnostic("Vertex received Kill while in RUNNING state.");
       for (Task task : vertex.tasks.values()) {
         vertex.eventHandler.handle(
             new TaskEvent(task.getTaskId(), TaskEventType.T_KILL));
@@ -1133,14 +1143,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
 
     @Override
     public VertexState transition(VertexImpl vertex, VertexEvent event) {
-      VertexState vertexCompleteSuccess =
-          VertexImpl.checkVertexForCompletion(vertex);
-      if (vertexCompleteSuccess != null) {
-        return vertexCompleteSuccess;
-      }
-
-      // Return the current state, Job not finished yet
-      return vertex.getInternalState();
+      return VertexImpl.checkVertexForCompletion(vertex);
     }
   }
 
@@ -1226,6 +1229,16 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     return appContext.getDAG();
   }
   
+  @VisibleForTesting
+  String getProcessorName() {
+    return this.processorName;
+  }
+
+  @VisibleForTesting
+  String getJavaOpts() {
+    return this.javaOpts;
+  }
+
   // TODO Eventually remove synchronization.
   @Override
   public synchronized List<InputSpec> getInputSpecList() {
@@ -1261,4 +1274,19 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     return outputSpecList;
   }
 
+  @VisibleForTesting
+  VertexOutputCommitter getVertexOutputCommitter() {
+    return this.committer;
+  }
+
+  @VisibleForTesting
+  // Only to be used for testing
+  void setVertexOutputCommitter(VertexOutputCommitter committer) {
+    this.committer = committer;
+  }
+
+  @VisibleForTesting
+  VertexScheduler getVertexScheduler() {
+    return this.vertexScheduler;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/38b410e3/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
new file mode 100644
index 0000000..a1e450c
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -0,0 +1,950 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.MRVertexOutputCommitter;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.SystemClock;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.DAGProtos.PlanEdgeConnectionPattern;
+import org.apache.tez.dag.api.DAGProtos.PlanEdgeSourceType;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.DagTypeConverters;
+import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.VertexLocationHint;
+import org.apache.tez.dag.api.impl.NullVertexOutputCommitter;
+import org.apache.tez.dag.api.impl.VertexContext;
+import org.apache.tez.dag.api.impl.VertexOutputCommitter;
+import org.apache.tez.dag.api.impl.VertexStatus.State;
+import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskHeartbeatHandler;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.VertexState;
+import org.apache.tez.dag.app.dag.event.DAGEvent;
+import org.apache.tez.dag.app.dag.event.DAGEventType;
+import org.apache.tez.dag.app.dag.event.VertexEvent;
+import org.apache.tez.dag.app.dag.event.VertexEventTaskAttemptCompleted;
+import org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted;
+import org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule;
+import org.apache.tez.dag.app.dag.event.VertexEventType;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.avro.HistoryEventType;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.engine.common.security.JobTokenIdentifier;
+import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
+import org.apache.tez.engine.records.TezDependentTaskCompletionEvent.Status;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestVertexImpl {
+
+  private static final Log LOG = LogFactory.getLog(TestVertexImpl.class);
+
+  private TezDAGID dagId;
+  private ApplicationAttemptId appAttemptId;
+  private DAGPlan dagPlan;
+  private Map<String, VertexImpl> vertices;
+  private Map<TezVertexID, VertexImpl> vertexIdMap;
+  private DrainDispatcher dispatcher;
+  private TaskAttemptListener taskAttemptListener;
+  private Credentials fsTokens;
+  private Token<JobTokenIdentifier> jobToken;
+  private Clock clock = new SystemClock();
+  private TaskHeartbeatHandler thh;
+  private AppContext appContext;
+  private VertexLocationHint vertexLocationHint;
+  private TezConfiguration conf = new TezConfiguration();
+  private Map<String, EdgeProperty> edges;
+
+  private VertexEventDispatcher vertexEventDispatcher;
+
+  private DagEventDispatcher dagEventDispatcher;
+
+  private class CountingVertexOutputCommitter extends
+      VertexOutputCommitter {
+
+    public int initCounter = 0;
+    public int setupCounter = 0;
+    public int commitCounter = 0;
+    public int abortCounter = 0;
+    private boolean throwError;
+    private boolean throwErrorOnAbort;
+    
+    public CountingVertexOutputCommitter(boolean throwError,
+        boolean throwOnAbort) {
+      this.throwError = throwError;
+      this.throwErrorOnAbort = throwOnAbort;
+    }
+
+    public CountingVertexOutputCommitter() {
+      this(false, false);
+    }
+    
+    @Override
+    public void init(VertexContext context) throws IOException {
+      ++initCounter;
+    }
+
+    @Override
+    public void setupVertex() throws IOException {
+      ++setupCounter;
+    }
+
+    @Override
+    public void commitVertex() throws IOException {
+      ++commitCounter;
+      if (throwError) {
+        throw new IOException("I can throwz exceptions in commit");
+      }
+    }
+
+    @Override
+    public void abortVertex(State finalState) throws IOException {
+      ++abortCounter;
+      if (throwErrorOnAbort) {
+        throw new IOException("I can throwz exceptions in abort");
+      }
+    }    
+  }
+  
+  private class DagEventDispatcher implements EventHandler<DAGEvent> {
+    @Override
+    public void handle(DAGEvent event) {
+    }
+  }
+
+  private class HistoryHandler implements EventHandler<DAGHistoryEvent> {
+    @Override
+    public void handle(DAGHistoryEvent event) {
+    }
+  }
+  
+  private class VertexEventDispatcher
+      implements EventHandler<VertexEvent> {
+
+    @Override
+    public void handle(VertexEvent event) {
+      VertexImpl vertex = vertexIdMap.get(event.getVertexId());
+      ((EventHandler<VertexEvent>) vertex).handle(event);
+    }
+  }
+
+
+  private DAGPlan createTestDAGPlan() {
+    LOG.info("Setting up dag plan");
+    DAGPlan dag = DAGPlan.newBuilder()
+        .setName("testverteximpl")
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex1")
+            .setType(PlanVertexType.NORMAL)
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host1")
+                .addRack("rack1")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(0)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("")
+                .setTaskModule("x1.y1")
+                .build()
+                )
+            .addOutEdgeId("e1")
+            .build()
+            )
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex2")
+            .setType(PlanVertexType.NORMAL)
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host2")
+                .addRack("rack2")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(2)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("")
+                .setTaskModule("x2.y2")
+                .build()
+                )
+            .addOutEdgeId("e2")
+            .build()
+            )
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex3")
+            .setType(PlanVertexType.NORMAL)
+            .setProcessorName("x3.y3")
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host3")
+                .addRack("rack3")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(2)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("foo")
+                .setTaskModule("x3.y3")
+                .build()
+                )
+            .addInEdgeId("e1")
+            .addInEdgeId("e2")
+            .addOutEdgeId("e3")
+            .addOutEdgeId("e4")
+            .build()
+            )
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex4")
+            .setType(PlanVertexType.NORMAL)
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host4")
+                .addRack("rack4")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(2)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("")
+                .setTaskModule("x4.y4")
+                .build()
+                )
+            .addInEdgeId("e3")
+            .addOutEdgeId("e5")
+            .build()
+            )
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex5")
+            .setType(PlanVertexType.NORMAL)
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host5")
+                .addRack("rack5")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(2)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("")
+                .setTaskModule("x5.y5")
+                .build()
+                )
+            .addInEdgeId("e4")
+            .addOutEdgeId("e6")
+            .build()
+            )
+        .addVertex(
+            VertexPlan.newBuilder()
+            .setName("vertex6")
+            .setType(PlanVertexType.NORMAL)
+            .addTaskLocationHint(
+                PlanTaskLocationHint.newBuilder()
+                .addHost("host6")
+                .addRack("rack6")
+                .build()
+                )
+            .setTaskConfig(
+                PlanTaskConfiguration.newBuilder()
+                .setNumTasks(2)
+                .setVirtualCores(4)
+                .setMemoryMb(1024)
+                .setJavaOpts("")
+                .setTaskModule("x6.y6")
+                .build()
+                )
+            .addInEdgeId("e5")
+            .addInEdgeId("e6")
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i3_v1")
+            .setInputVertexName("vertex1")
+            .setOutputClass("o1")
+            .setOutputVertexName("vertex3")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e1")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i3_v2")
+            .setInputVertexName("vertex2")
+            .setOutputClass("o2")
+            .setOutputVertexName("vertex3")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e2")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i4_v3")
+            .setInputVertexName("vertex3")
+            .setOutputClass("o3_v4")
+            .setOutputVertexName("vertex4")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e3")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i5_v3")
+            .setInputVertexName("vertex3")
+            .setOutputClass("o3_v5")
+            .setOutputVertexName("vertex5")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e4")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i6_v4")
+            .setInputVertexName("vertex4")
+            .setOutputClass("o4")
+            .setOutputVertexName("vertex6")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e5")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .addEdge(
+            EdgePlan.newBuilder()
+            .setInputClass("i6_v5")
+            .setInputVertexName("vertex5")
+            .setOutputClass("o5")
+            .setOutputVertexName("vertex6")
+            .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+            .setId("e6")
+            .setSourceType(PlanEdgeSourceType.STABLE)
+            .build()
+            )
+        .build();
+
+    return dag;
+  }
+
+  private void setupVertices() {
+    int vCnt = dagPlan.getVertexCount();
+    LOG.info("Setting up vertices from dag plan, verticesCnt=" + vCnt);
+    vertices = new HashMap<String, VertexImpl>();
+    vertexIdMap = new HashMap<TezVertexID, VertexImpl>();
+    for (int i = 0; i < vCnt; ++i) {
+      VertexPlan vPlan = dagPlan.getVertex(i);
+      TezVertexID vertexId = new TezVertexID(dagId, i+1);
+      VertexImpl v = new VertexImpl(vertexId, vPlan, vPlan.getName(), conf,
+          dispatcher.getEventHandler(), taskAttemptListener, jobToken, fsTokens,
+          clock, thh, appContext, vertexLocationHint);
+      vertices.put(vPlan.getName(), v);
+      vertexIdMap.put(vertexId, v);
+    }
+  }
+
+  private void parseVertexEdges() {
+    LOG.info("Parsing edges from dag plan, edgeCount="
+        + dagPlan.getEdgeCount());
+    int vCnt = dagPlan.getVertexCount();
+    Map<String, EdgePlan> edgePlans =
+        DagTypeConverters.createEdgePlanMapFromDAGPlan(dagPlan.getEdgeList());
+
+    for (int i = 0; i < vCnt; ++i) {
+      VertexPlan vertexPlan = dagPlan.getVertex(i);
+      Vertex vertex = vertices.get(vertexPlan.getName());
+      Map<Vertex, EdgeProperty> inVertices =
+          new HashMap<Vertex, EdgeProperty>();
+
+      Map<Vertex, EdgeProperty> outVertices =
+          new HashMap<Vertex, EdgeProperty>();
+      
+      for(String inEdgeId : vertexPlan.getInEdgeIdList()){
+        EdgePlan edgePlan = edgePlans.get(inEdgeId);
+        Vertex inVertex = this.vertices.get(edgePlan.getInputVertexName());
+        EdgeProperty edgeProp = this.edges.get(inEdgeId);
+        inVertices.put(inVertex, edgeProp);
+      }
+
+      for(String outEdgeId : vertexPlan.getOutEdgeIdList()){
+        EdgePlan edgePlan = edgePlans.get(outEdgeId);
+        Vertex outVertex = this.vertices.get(edgePlan.getOutputVertexName());
+        EdgeProperty edgeProp = this.edges.get(outEdgeId);
+        outVertices.put(outVertex, edgeProp);
+      }
+      LOG.info("Setting input vertices for vertex " + vertex.getName()
+          + ", inputVerticesCnt=" + inVertices.size());
+      vertex.setInputVertices(inVertices);
+      LOG.info("Setting output vertices for vertex " + vertex.getName()
+          + ", outputVerticesCnt=" + outVertices.size());
+      vertex.setOutputVertices(outVertices);
+    }
+  }
+
+  @Before
+  public void setup() {
+    appAttemptId = BuilderUtils.newApplicationAttemptId(
+        BuilderUtils.newApplicationId(100, 1), 1);
+    dagId = new TezDAGID(appAttemptId.getApplicationId(), 1);
+    dagPlan = createTestDAGPlan();
+    dispatcher = new DrainDispatcher();
+    fsTokens = new Credentials();
+    jobToken = new Token<JobTokenIdentifier>();
+    appContext = mock(AppContext.class);
+    DAG dag = mock(DAG.class);
+    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
+    doReturn(dag).when(appContext).getDAG();
+    doReturn(dagId).when(appContext).getDAGID();
+    doReturn(dagId).when(dag).getID();
+    setupVertices();
+    edges = DagTypeConverters.createEdgePropertyMapFromDAGPlan(
+        dagPlan.getEdgeList());
+    parseVertexEdges();
+    vertexEventDispatcher = new VertexEventDispatcher();
+    dispatcher.register(VertexEventType.class, vertexEventDispatcher);
+    dagEventDispatcher = new DagEventDispatcher();
+    dispatcher.register(DAGEventType.class, dagEventDispatcher);
+    dispatcher.register(HistoryEventType.class,
+        new HistoryHandler());
+    dispatcher.init(conf);
+    dispatcher.start();
+  }
+
+  @After
+  public void teardown() {
+    dagPlan = null;
+    this.vertices = null;
+    this.edges = null;
+    dispatcher.await();
+    dispatcher.stop();
+  }
+
+  private void initVertex(VertexImpl v) {
+    Assert.assertEquals(VertexState.NEW, v.getState());
+    v.handle(new VertexEvent(v.getVertexId(),
+          VertexEventType.V_INIT));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.INITED, v.getState());
+  }
+
+  private void startVertex(VertexImpl v) {
+    startVertex(v, true);
+  }
+
+  private void killVertex(VertexImpl v, boolean checkKillWait) {
+    v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+    dispatcher.await();
+    if (checkKillWait) {
+      Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+    } else {
+      Assert.assertEquals(VertexState.KILLED, v.getState());      
+    }
+  }
+  
+  private void startVertex(VertexImpl v,
+      boolean checkRunningState) {
+    Assert.assertEquals(VertexState.INITED, v.getState());
+    v.handle(new VertexEvent(v.getVertexId(),
+          VertexEventType.V_START));
+    dispatcher.await();
+    if (checkRunningState) {
+      Assert.assertEquals(VertexState.RUNNING, v.getState());
+    }
+  }
+
+  @Test
+  public void testVertexInit() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+
+    VertexImpl v3 = vertices.get("vertex3");
+    initVertex(v3);
+    
+    Assert.assertEquals("x3.y3", v3.getProcessorName());
+    Assert.assertEquals("foo", v3.getJavaOpts());
+    
+    Assert.assertEquals(2, v3.getInputSpecList().size());
+    Assert.assertEquals(2, v3.getInputVerticesCount());
+    Assert.assertEquals(2, v3.getOutputVerticesCount());
+    Assert.assertEquals(2, v3.getOutputVerticesCount());
+
+    Assert.assertTrue("vertex1".equals(v3.getInputSpecList().get(0)
+        .getVertexName())
+        || "vertex2".equals(v3.getInputSpecList().get(0)
+            .getVertexName()));
+    Assert.assertTrue("vertex1".equals(v3.getInputSpecList().get(1)
+        .getVertexName())
+        || "vertex2".equals(v3.getInputSpecList().get(1)
+            .getVertexName()));
+    Assert.assertTrue("i3_v1".equals(v3.getInputSpecList().get(0)
+        .getInputClassName())
+        || "i3_v2".equals(v3.getInputSpecList().get(0)
+            .getInputClassName()));
+    Assert.assertTrue("i3_v1".equals(v3.getInputSpecList().get(1)
+        .getInputClassName())
+        || "i3_v2".equals(v3.getInputSpecList().get(1)
+            .getInputClassName()));
+    
+    Assert.assertTrue("vertex4".equals(v3.getOutputSpecList().get(0)
+        .getVertexName())
+        || "vertex5".equals(v3.getOutputSpecList().get(0)
+            .getVertexName()));
+    Assert.assertTrue("vertex4".equals(v3.getOutputSpecList().get(1)
+        .getVertexName())
+        || "vertex5".equals(v3.getOutputSpecList().get(1)
+            .getVertexName()));
+    Assert.assertTrue("o3_v4".equals(v3.getOutputSpecList().get(0)
+        .getOutputClassName())
+        || "o3_v5".equals(v3.getOutputSpecList().get(0)
+            .getOutputClassName()));
+    Assert.assertTrue("o3_v4".equals(v3.getOutputSpecList().get(1)
+        .getOutputClassName())
+        || "o3_v5".equals(v3.getOutputSpecList().get(1)
+            .getOutputClassName()));
+  }
+
+  @Test
+  public void testVertexStart() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    startVertex(v);
+  }
+
+  @Test
+  public void testBasicVertexCompletion() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    startVertex(v);
+
+    TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+    TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.RUNNING, v.getState());
+    Assert.assertEquals(1, v.getCompletedTasks());
+
+    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+    Assert.assertEquals(2, v.getCompletedTasks());
+  }
+
+  @Test
+  @Ignore // FIXME fix verteximpl for this test to work
+  public void testDuplicateTaskCompletion() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    startVertex(v);
+
+    TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+    TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.RUNNING, v.getState());
+    
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.RUNNING, v.getState());
+
+    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+  }
+
+
+  @Test
+  public void testVertexFailure() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    startVertex(v);
+
+    TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.FAILED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.FAILED, v.getState());
+    String diagnostics =
+        StringUtils.join(",", v.getDiagnostics()).toLowerCase();
+    Assert.assertTrue(diagnostics.contains("task failed " + t1.toString()));
+  }
+  
+  @Test
+  public void testVertexWithNoTasks() {
+    // FIXME a vertex with no tasks should not be allowed
+    VertexImpl v = vertices.get("vertex1");
+    initVertex(v);
+    startVertex(v, false);
+    dispatcher.await();
+    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+  }
+
+  @Test
+  public void testVertexKill() {
+    VertexImpl v1 = vertices.get("vertex1");
+    killVertex(v1, false);
+    String diagnostics =
+        StringUtils.join(",", v1.getDiagnostics()).toLowerCase();
+    Assert.assertTrue(diagnostics.contains(
+        "vertex received kill in new state"));
+    
+    VertexImpl v2 = vertices.get("vertex2");
+    initVertex(v2);
+    killVertex(v2, false);
+    diagnostics =
+        StringUtils.join(",", v2.getDiagnostics()).toLowerCase();
+    LOG.info("diagnostics v2: " + diagnostics);
+    Assert.assertTrue(diagnostics.contains(
+        "vertex received kill in inited state"));
+
+    VertexImpl v3 = vertices.get("vertex3");
+    initVertex(v3);
+    startVertex(v3);
+    killVertex(v3, true);
+    diagnostics =
+        StringUtils.join(",", v3.getDiagnostics()).toLowerCase();
+    Assert.assertTrue(diagnostics.contains(
+        "vertex received kill while in running state"));
+  }
+
+  @Test
+  public void testKilledTasksHandling() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    startVertex(v);
+
+    TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+    TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.KILLED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.RUNNING, v.getState());
+
+    v.handle(new VertexEventTaskCompleted(t2, TaskState.KILLED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.KILLED, v.getState());
+  }
+
+  @Test
+  public void testVertexCommitterInit() {
+    VertexImpl v2 = vertices.get("vertex2");
+    initVertex(v2);
+    Assert.assertTrue(v2.getVertexOutputCommitter()
+        instanceof NullVertexOutputCommitter);
+    
+    VertexImpl v6 = vertices.get("vertex6");
+    initVertex(v6);
+    Assert.assertTrue(v6.getVertexOutputCommitter()
+        instanceof MRVertexOutputCommitter);
+  }
+
+  @Test
+  public void testVertexSchedulerInit() {
+    VertexImpl v2 = vertices.get("vertex2");
+    initVertex(v2);
+    Assert.assertTrue(v2.getVertexScheduler()
+        instanceof ImmediateStartVertexScheduler);
+    
+    VertexImpl v6 = vertices.get("vertex6");
+    initVertex(v6);
+    Assert.assertTrue(v6.getVertexScheduler()
+        instanceof BipartiteSlowStartVertexScheduler);
+  }
+  
+  @Test
+  public void testVertexTaskFailure() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    CountingVertexOutputCommitter committer =
+        new CountingVertexOutputCommitter();
+    v.setVertexOutputCommitter(committer);
+    startVertex(v);
+
+    TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+    TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.RUNNING, v.getState());
+    
+    v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
+    v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.FAILED, v.getState());
+    Assert.assertEquals(0, committer.commitCounter);
+    Assert.assertEquals(1, committer.abortCounter);
+  }
+
+  @Test
+  public void testSourceVertexStartHandling() {
+    VertexImpl v4 = vertices.get("vertex4");
+    initVertex(v4);
+    VertexImpl v5 = vertices.get("vertex5");
+    initVertex(v5);
+    VertexImpl v6 = vertices.get("vertex6");
+    initVertex(v6);
+    Assert.assertEquals(VertexState.INITED, v6.getState());
+
+    startVertex(v4);
+    startVertex(v5);
+    dispatcher.await();
+    Assert.assertEquals(VertexState.RUNNING, v6.getState());
+    Assert.assertEquals(1, v6.getDistanceFromRoot());
+  }
+
+  @Test
+  public void testCounters() {
+    // FIXME need to test counters at vertex level
+  }
+
+  @Test
+  public void testDiagnostics() {
+    // FIXME need to test diagnostics in various cases
+  }
+  
+  @Test
+  public void testTaskAttemptCompletionEvents() {    
+    // FIXME need to test handling of task attempt events
+  }
+
+  @Test
+  public void testSourceTaskAttemptCompletionEvents() {
+    VertexImpl v4 = vertices.get("vertex4");
+    initVertex(v4);
+    VertexImpl v5 = vertices.get("vertex5");
+    initVertex(v5);
+    VertexImpl v6 = vertices.get("vertex6");
+    initVertex(v6);
+
+    startVertex(v4);
+    startVertex(v5);
+    dispatcher.await();
+    Assert.assertEquals(VertexState.RUNNING, v6.getState());
+
+    TezTaskID t1_v4 = new TezTaskID(v4.getVertexId(), 0);
+    TezTaskID t2_v4 = new TezTaskID(v4.getVertexId(), 1);
+    TezTaskID t1_v5 = new TezTaskID(v5.getVertexId(), 0);
+    TezTaskID t2_v5 = new TezTaskID(v5.getVertexId(), 1);
+
+    TezTaskAttemptID ta1_t1_v4 = new TezTaskAttemptID(t1_v4, 0);
+    TezTaskAttemptID ta2_t1_v4 = new TezTaskAttemptID(t1_v4, 0);
+    TezTaskAttemptID ta1_t2_v4 = new TezTaskAttemptID(t2_v4, 0);
+    TezTaskAttemptID ta1_t1_v5 = new TezTaskAttemptID(t1_v5, 0);
+    TezTaskAttemptID ta1_t2_v5 = new TezTaskAttemptID(t2_v5, 0);
+    TezTaskAttemptID ta2_t2_v5 = new TezTaskAttemptID(t2_v5, 0);
+
+    TezDependentTaskCompletionEvent cEvt1 =
+        new TezDependentTaskCompletionEvent(1, ta1_t1_v4,
+            Status.FAILED, "", 3);
+    TezDependentTaskCompletionEvent cEvt2 =
+        new TezDependentTaskCompletionEvent(2, ta2_t1_v4,
+            Status.SUCCEEDED, "", 4);
+    TezDependentTaskCompletionEvent cEvt3 =
+        new TezDependentTaskCompletionEvent(2, ta1_t2_v4,
+            Status.SUCCEEDED, "", 5);
+    TezDependentTaskCompletionEvent cEvt4 =
+        new TezDependentTaskCompletionEvent(2, ta1_t1_v5,
+            Status.SUCCEEDED, "", 5);
+    TezDependentTaskCompletionEvent cEvt5 =
+        new TezDependentTaskCompletionEvent(1, ta1_t2_v5,
+            Status.FAILED, "", 3);
+    TezDependentTaskCompletionEvent cEvt6 =
+        new TezDependentTaskCompletionEvent(2, ta2_t2_v5,
+            Status.SUCCEEDED, "", 4);
+
+    v4.handle(new VertexEventTaskAttemptCompleted(cEvt1));
+    v4.handle(new VertexEventTaskAttemptCompleted(cEvt2));
+    v4.handle(new VertexEventTaskAttemptCompleted(cEvt3));
+    v5.handle(new VertexEventTaskAttemptCompleted(cEvt4));
+    v5.handle(new VertexEventTaskAttemptCompleted(cEvt5));
+    v5.handle(new VertexEventTaskAttemptCompleted(cEvt6));    
+
+    v4.handle(new VertexEventTaskCompleted(t1_v4, TaskState.SUCCEEDED));
+    v4.handle(new VertexEventTaskCompleted(t2_v4, TaskState.SUCCEEDED));
+    v5.handle(new VertexEventTaskCompleted(t1_v5, TaskState.SUCCEEDED));
+    v5.handle(new VertexEventTaskCompleted(t2_v5, TaskState.SUCCEEDED));
+    dispatcher.await();
+
+    Assert.assertEquals(VertexState.SUCCEEDED, v4.getState());
+    Assert.assertEquals(VertexState.SUCCEEDED, v5.getState());
+
+    Assert.assertEquals(VertexState.RUNNING, v6.getState());
+    Assert.assertEquals(4, v6.successSourceAttemptCompletionEventNoMap.size());
+    Assert.assertEquals(6, v6.getTaskAttemptCompletionEvents(0, 100).length);
+  }
+
+  @Test
+  public void testDAGEventGeneration() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    startVertex(v);
+
+    TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+    TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+
+  }
+
+  @Test
+  public void testTaskReschedule() {
+    // For downstream failures
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    CountingVertexOutputCommitter committer =
+        new CountingVertexOutputCommitter();
+    v.setVertexOutputCommitter(committer);
+    
+    startVertex(v);
+
+    TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+    TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    v.handle(new VertexEventTaskReschedule(t1));
+    // FIXME need to handle dups
+    // v.handle(new VertexEventTaskReschedule(t1));
+    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.RUNNING, v.getState());    
+    Assert.assertEquals(0, committer.commitCounter);
+    
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());    
+    Assert.assertEquals(1, committer.commitCounter);
+    
+  }
+
+  @Test
+  public void testVertexCommit() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    CountingVertexOutputCommitter committer =
+        new CountingVertexOutputCommitter();
+    v.setVertexOutputCommitter(committer);
+    
+    startVertex(v);
+
+    TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+    TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());    
+    Assert.assertEquals(1, committer.commitCounter);
+    
+    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());    
+    Assert.assertEquals(1, committer.commitCounter);
+    Assert.assertEquals(0, committer.abortCounter);
+    Assert.assertEquals(0, committer.initCounter); // already done in init
+    Assert.assertEquals(0, committer.setupCounter); // already done in init
+  }
+  
+  @Test
+  public void testCommitterInitAndSetup() {
+    // FIXME need to add a test for this
+  }
+  
+  @Test
+  public void testTaskAttemptFetchFailureHandling() {
+    // FIXME needs testing
+  }
+
+  @Test
+  public void testBadCommitter() {
+    VertexImpl v = vertices.get("vertex2");
+    initVertex(v);
+    CountingVertexOutputCommitter committer =
+        new CountingVertexOutputCommitter(true, true);
+    v.setVertexOutputCommitter(committer);
+    
+    startVertex(v);
+
+    TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+    TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+    v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+    v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+    dispatcher.await();
+    Assert.assertEquals(VertexState.FAILED, v.getState());    
+    Assert.assertEquals(1, committer.commitCounter);
+    
+    // FIXME need to verify whether abort needs to be called if commit fails
+    Assert.assertEquals(0, committer.abortCounter);
+    Assert.assertEquals(0, committer.initCounter); // already done in init
+    Assert.assertEquals(0, committer.setupCounter); // already done in init  
+  }
+  
+}


[44/50] [abbrv] git commit: Fix build after YARN-717, YARN-720. (sseth)

Posted by ss...@apache.org.
Fix build after YARN-717, YARN-720. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/86523292
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/86523292
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/86523292

Branch: refs/heads/master
Commit: 8652329273978266490806028a4c274af33d213e
Parents: eeaaf23
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jun 3 09:25:46 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jun 3 09:25:46 2013 -0700

----------------------------------------------------------------------
 .../dag/app/launcher/ContainerLauncherImpl.java    |   13 ++++++-------
 .../apache/tez/dag/app/rm/NMCommunicatorEvent.java |    8 ++++----
 .../dag/app/rm/NMCommunicatorStopRequestEvent.java |    4 ++--
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |    7 ++-----
 4 files changed, 14 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index 5b23c75..b71af02 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.mapred.ShuffleHandler;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.YarnException;
@@ -46,7 +45,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
 import org.apache.hadoop.yarn.service.AbstractService;
@@ -123,10 +122,10 @@ public class ContainerLauncherImpl extends AbstractService implements
     // store enough information to be able to cleanup the container
     private ContainerId containerID;
     final private String containerMgrAddress;
-    private ContainerToken containerToken;
+    private Token containerToken;
     
     public Container(ContainerId containerID,
-        String containerMgrAddress, ContainerToken containerToken) {
+        String containerMgrAddress, Token containerToken) {
       this.state = ContainerState.PREP;
       this.containerMgrAddress = containerMgrAddress;
       this.containerID = containerID;
@@ -355,7 +354,7 @@ public class ContainerLauncherImpl extends AbstractService implements
   }
 
   protected ContainerManager getCMProxy(ContainerId containerID,
-      final String containerManagerBindAddr, ContainerToken containerToken)
+      final String containerManagerBindAddr, Token containerToken)
       throws IOException {
 
     final InetSocketAddress cmAddr =
@@ -364,8 +363,8 @@ public class ContainerLauncherImpl extends AbstractService implements
     // the user in createRemoteUser in this context has to be ContainerID
     UserGroupInformation user = UserGroupInformation
         .createRemoteUser(containerID.toString());
-    Token<ContainerTokenIdentifier> token = ProtoUtils.convertFromProtoFormat(
-        containerToken, cmAddr);
+    org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token = ProtoUtils
+        .convertFromProtoFormat(containerToken, cmAddr);
     user.addToken(token);
 
     ContainerManager proxy = user

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
index 38f54c6..8bdeb28 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
@@ -19,18 +19,18 @@
 package org.apache.tez.dag.app.rm;
 
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.event.AbstractEvent;
 
 public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType> {
 
   private final ContainerId containerId;
   private final NodeId nodeId;
-  private final ContainerToken containerToken;
+  private final Token containerToken;
 
   public NMCommunicatorEvent(ContainerId containerId, NodeId nodeId,
-      ContainerToken containerToken, NMCommunicatorEventType type) {
+      Token containerToken, NMCommunicatorEventType type) {
     super(type);
     this.containerId = containerId;
     this.nodeId = nodeId;
@@ -45,7 +45,7 @@ public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType>
     return this.nodeId;
   }
 
-  public ContainerToken getContainerToken() {
+  public Token getContainerToken() {
     return this.containerToken;
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
index 8f09a48..277d1e7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
@@ -19,13 +19,13 @@
 package org.apache.tez.dag.app.rm;
 
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Token;
 
 public class NMCommunicatorStopRequestEvent extends NMCommunicatorEvent {
 
   public NMCommunicatorStopRequestEvent(ContainerId containerId, NodeId nodeId,
-      ContainerToken containerToken) {
+      Token containerToken) {
     super(containerId, nodeId, containerToken,
         NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 801ce8a..572437b 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -70,7 +70,6 @@ import org.apache.hadoop.mapreduce.v2.LogParams;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.YarnException;
@@ -1141,13 +1140,11 @@ public class YARNRunner implements ClientProtocol {
   private static void addLog4jSystemProperties(String logLevel,
       List<String> vargs) {
     vargs.add("-Dlog4j.configuration=container-log4j.properties");
-    // TODO Fix the remaining properties after YARN-720 is fixed.
-    // May need to introduce a log4j properties file for tez tasks.
-    vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "="
+    vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
         + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
     // Setting this to 0 to avoid log size restrictions.
     // Should be enforced by YARN.
-    vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + 0);
+    vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=" + 0);
     vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
   }
 


[27/50] [abbrv] git commit: TEZ-155. Basic fixes for Hive (gunther via bikas)

Posted by ss...@apache.org.
TEZ-155. Basic fixes for Hive (gunther via bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/5d8adeb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/5d8adeb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/5d8adeb5

Branch: refs/heads/master
Commit: 5d8adeb52d1ef46b4ea3874a675ace930c45bc91
Parents: 4c2732c
Author: Bikas Saha <bi...@apache.org>
Authored: Thu May 23 18:34:03 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Thu May 23 18:34:03 2013 -0700

----------------------------------------------------------------------
 .../tez/mapreduce/ClientServiceDelegate.java       |   21 ++++++++++-----
 .../org/apache/tez/mapreduce/DAGJobStatus.java     |   11 ++++---
 2 files changed, 20 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5d8adeb5/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
index 44ccc07..1335a44 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
@@ -24,7 +24,9 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.mapreduce.*;
 import org.apache.hadoop.mapreduce.v2.LogParams;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.mapreduce.TaskCompletionEvent;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -87,12 +89,17 @@ public class ClientServiceDelegate {
   }
   
   public JobStatus getJobStatus(JobID oldJobID) throws IOException {
+    org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
+      TypeConverter.toYarn(oldJobID);
+    String user = UserGroupInformation.getCurrentUser().getShortUserName();
+    String jobFile = MRApps.getJobFile(conf, user, oldJobID);
+
     try {
       if(dagClient == null) {
         appReport = getAppReport(oldJobID);
         if(appReport.getYarnApplicationState() != YarnApplicationState.RUNNING) {
           // if job not running return status from appReport;
-          return getJobStatusFromRM(appReport);
+          return getJobStatusFromRM(appReport, jobFile);
         } else {
           // job is running. create dag am client
           dagClient = tezClient.getDAGClient(appReport.getHost(), 
@@ -102,7 +109,7 @@ public class ClientServiceDelegate {
       }
       // return status from client. use saved appReport for queue etc
       DAGStatus dagStatus = dagClient.getDAGStatus(currentDAGId);
-      return new DAGJobStatus(appReport, dagStatus);
+      return new DAGJobStatus(appReport, dagStatus, jobFile);
     } catch (TezRemoteException e) {
       // AM not responding
       dagClient = null;
@@ -116,20 +123,20 @@ public class ClientServiceDelegate {
     }
     
     // final fallback
-    return getJobStatusFromRM(appReport);
+    return getJobStatusFromRM(appReport, jobFile);
   }
   
-  private JobStatus getJobStatusFromRM(ApplicationReport appReport) {
+  private JobStatus getJobStatusFromRM(ApplicationReport appReport, String jobFile) {
     JobStatus jobStatus =
-        new DAGJobStatus(appReport, null);
+      new DAGJobStatus(appReport, null, jobFile);
     return jobStatus;            
   }
 
   public org.apache.hadoop.mapreduce.TaskReport[] getTaskReports(
       JobID oldJobID, TaskType taskType)
        throws IOException{
-    // FIXME need support to query task reports?
-    throw new UnsupportedOperationException();
+    // TEZ-146: need to return real task reports
+    return new org.apache.hadoop.mapreduce.TaskReport[0];
   }
 
   public boolean killTask(TaskAttemptID taskAttemptID, boolean fail)

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5d8adeb5/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
index 08c61f8..10d05b3 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
@@ -39,13 +39,15 @@ import org.mortbay.log.Log;
 
 public class DAGJobStatus extends JobStatus {
 
+  String jobFile;
   private final ApplicationReport report;
   private final DAGStatus dagStatus;
   
-  public DAGJobStatus(ApplicationReport appReport, DAGStatus dagStatus) {
+  public DAGJobStatus(ApplicationReport appReport, DAGStatus dagStatus, String jobFile) {
     super();
     this.report = appReport;
     this.dagStatus = dagStatus;
+    this.jobFile = jobFile;
   }
   
   @Override
@@ -215,8 +217,8 @@ public class DAGJobStatus extends JobStatus {
 
   @Override
   public synchronized JobPriority getPriority() {
-    // TODO Auto-generated method stub
-    return super.getPriority();
+    // TEX-147: return real priority
+    return JobPriority.NORMAL;
   }
 
   @Override
@@ -251,8 +253,7 @@ public class DAGJobStatus extends JobStatus {
 
   @Override
   public String getJobFile() {
-    // FIXME
-    throw new UnsupportedOperationException();
+    return jobFile;
   }
 
   @Override


[47/50] [abbrv] git commit: TEZ-175. Cleanup logs.

Posted by ss...@apache.org.
TEZ-175. Cleanup logs.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/9feab053
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/9feab053
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/9feab053

Branch: refs/heads/master
Commit: 9feab053fc0508f9ec6b791db8afdd4eebc11113
Parents: 9f040cf
Author: Hitesh Shah <hi...@apache.org>
Authored: Mon Jun 3 16:42:28 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Mon Jun 3 16:46:40 2013 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/mapred/YarnTezDagChild.java  |   31 ++--
 .../java/org/apache/tez/dag/app/DAGAppMaster.java  |  102 +++++-----
 .../tez/dag/app/TaskAttemptListenerImpTezDag.java  |   60 +++---
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |   60 +++---
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java      |  104 +++++-----
 .../org/apache/tez/dag/app/dag/impl/TaskImpl.java  |  164 ++++++++-------
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |   36 ++--
 .../tez/dag/app/rm/container/AMContainerImpl.java  |   89 +++++----
 .../org/apache/tez/dag/app/rm/node/AMNodeImpl.java |   24 ++-
 .../engine/common/shuffle/impl/InMemoryWriter.java |   46 ++--
 .../engine/common/sort/impl/ExternalSorter.java    |   38 ++--
 .../engine/common/sort/impl/IFileOutputStream.java |   14 +-
 .../common/sort/impl/dflt/DefaultSorter.java       |   88 ++++----
 .../task/local/output/TezTaskOutputFiles.java      |   38 ++--
 .../engine/lib/output/LocalOnFileSorterOutput.java |   12 +-
 .../apache/tez/engine/runtime/RuntimeUtils.java    |   29 ++-
 .../org/apache/hadoop/mapred/LocalJobRunner.java   |  122 ++++++------
 .../mapreduce/split/SplitMetaInfoReaderTez.java    |   12 +-
 .../apache/tez/mapreduce/combine/MRCombiner.java   |    7 +-
 .../hadoop/MultiStageMRConfToTezTranslator.java    |   34 ++--
 .../org/apache/tez/mapreduce/processor/MRTask.java |  147 +++++++-------
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |   64 ++++---
 22 files changed, 686 insertions(+), 635 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 6fd6eff..33e3972 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -146,9 +146,10 @@ public class YarnTezDagChild {
         }
         taskContext = (TezEngineTaskContext) containerTask
             .getTezEngineTaskContext();
-        LOG.info("DEBUG: New container task context:"
-                + taskContext.toString());
-
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("New container task context:"
+              + taskContext.toString());
+        }
         taskAttemptId = taskContext.getTaskAttemptId();
 
         final Task t = createAndConfigureTezTask(taskContext, umbilical,
@@ -200,25 +201,25 @@ public class YarnTezDagChild {
   /**
    * Configure mapred-local dirs. This config is used by the task for finding
    * out an output directory.
-   * @throws IOException 
+   * @throws IOException
    */
   /**
    * Configure tez-local-dirs, tez-localized-file-dir, etc. Also create these
    * dirs.
    */
-  
+
   private static void configureLocalDirs(Configuration conf) throws IOException {
     String[] localSysDirs = StringUtils.getTrimmedStrings(
         System.getenv(Environment.LOCAL_DIRS.name()));
     conf.setStrings(TezJobConfig.LOCAL_DIRS, localSysDirs);
     conf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
         System.getenv(Environment.PWD.name()));
-    
+
     LOG.info(TezJobConfig.LOCAL_DIRS + " for child: " +
         conf.get(TezJobConfig.LOCAL_DIRS));
     LOG.info(TezJobConfig.TASK_LOCAL_RESOURCE_DIR + " for child: "
         + conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
-    
+
     LocalDirAllocator lDirAlloc = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
     Path workDir = null;
     // First, try to find the JOB_LOCAL_DIR on this host.
@@ -250,7 +251,7 @@ public class YarnTezDagChild {
     }
     conf.set(TezJobConfig.JOB_LOCAL_DIR, workDir.toString());
   }
-  
+
   private static Task createAndConfigureTezTask(
       TezEngineTaskContext taskContext, TezTaskUmbilicalProtocol master,
       Credentials cxredentials, Token<JobTokenIdentifier> jobToken,
@@ -260,10 +261,10 @@ public class YarnTezDagChild {
     // set tcp nodelay
     conf.setBoolean("ipc.client.tcpnodelay", true);
     conf.setInt(TezJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId);
-    
+
     configureLocalDirs(conf);
-    
-    
+
+
     // FIXME need Input/Output vertices else we have this hack
     if (taskContext.getInputSpecList().isEmpty()) {
       taskContext.getInputSpecList().add(
@@ -281,18 +282,18 @@ public class YarnTezDagChild {
     // and processor then inits inputs and outputs
     return t;
   }
-  
+
   private static void runTezTask(
-      Task t, TezTaskUmbilicalProtocol master, Configuration conf) 
+      Task t, TezTaskUmbilicalProtocol master, Configuration conf)
   throws IOException, InterruptedException {
     // use job-specified working directory
     FileSystem.get(conf).setWorkingDirectory(getWorkingDirectory(conf));
-    
+
     // Run!
     t.run();
     t.close();
   }
-  
+
   private static Path getWorkingDirectory(Configuration conf) {
     String name = conf.get(JobContext.WORKING_DIR);
     if (name != null) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 821dd71..a4ca97e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -128,7 +128,7 @@ import org.apache.tez.engine.common.security.JobTokenSecretManager;
 
 @SuppressWarnings("rawtypes")
 public class DAGAppMaster extends CompositeService {
-  
+
   private static final Log LOG = LogFactory.getLog(DAGAppMaster.class);
 
   /**
@@ -137,7 +137,7 @@ public class DAGAppMaster extends CompositeService {
   public static final int SHUTDOWN_HOOK_PRIORITY = 30;
 
   private Clock clock;
-  private final DAGPlan jobPlan;
+  private final DAGPlan dagPlan;
   private long dagsStartTime;
   private final long startTime;
   private final long appSubmitTime;
@@ -154,7 +154,7 @@ public class DAGAppMaster extends CompositeService {
   // TODO Recovery
   //private Map<TezTaskID, TaskInfo> completedTasksFromPreviousRun;
   private AppContext context;
-  private TezConfiguration conf; 
+  private TezConfiguration conf;
   private Dispatcher dispatcher;
   // TODO Recovery
   //private Recovery recoveryServ;
@@ -176,7 +176,7 @@ public class DAGAppMaster extends CompositeService {
   private HistoryEventHandler historyEventHandler;
 
   private DAGAppMasterState state;
-  
+
   DAGClientServer clientRpcServer;
   private DAGClientHandler clientHandler;
 
@@ -195,7 +195,7 @@ public class DAGAppMaster extends CompositeService {
       ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
       Clock clock, long appSubmitTime, DAGPlan dagPB) {
     super(DAGAppMaster.class.getName());
-    this.jobPlan = dagPB;
+    this.dagPlan = dagPB;
     this.clock = clock;
     this.startTime = clock.getTime();
     this.appSubmitTime = appSubmitTime;
@@ -214,9 +214,9 @@ public class DAGAppMaster extends CompositeService {
   public void init(final Configuration tezConf) {
 
     this.state = DAGAppMasterState.INITED;
-    
+
     assert tezConf instanceof TezConfiguration;
-    
+
     this.conf = (TezConfiguration) tezConf;
     conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
 
@@ -226,10 +226,10 @@ public class DAGAppMaster extends CompositeService {
 
     // Job name is the same as the app name util we support DAG of jobs
     // for an app later
-    appName = jobPlan.getName();
+    appName = dagPlan.getName();
 
     dagId = new TezDAGID(appAttemptID.getApplicationId(), 1);
-    
+
     clientHandler = new DAGClientHandler();
 
     // TODO Committer.
@@ -237,7 +237,7 @@ public class DAGAppMaster extends CompositeService {
 
     dispatcher = createDispatcher();
     addIfService(dispatcher);
-    
+
     clientRpcServer = new DAGClientServer(clientHandler);
     addIfService(clientRpcServer);
 
@@ -432,7 +432,7 @@ public class DAGAppMaster extends CompositeService {
       LOG.warn("No handler for event type: " + event.getType());
     }
   }
-  
+
   private class DAGAppMasterEventHandler implements
       EventHandler<DAGAppMasterEvent> {
     @Override
@@ -440,7 +440,7 @@ public class DAGAppMaster extends CompositeService {
       DAGAppMaster.this.handle(event);
     }
   }
-  
+
   private class DAGFinishEventHandler implements EventHandler<DAGFinishEvent> {
     @Override
     public void handle(DAGFinishEvent event) {
@@ -466,9 +466,9 @@ public class DAGAppMaster extends CompositeService {
     */
       // TODO:currently just wait for some time so clients can know the
       // final states. Will be removed once RM come on.
-     
+
       setStateOnDAGCompletion();
-      
+
       try {
         Thread.sleep(5000);
       } catch (InterruptedException e) {
@@ -660,19 +660,19 @@ public class DAGAppMaster extends CompositeService {
   public TaskAttemptListener getTaskAttemptListener() {
     return taskAttemptListener;
   }
-  
+
   public ContainerId getAppContainerId() {
     return containerID;
   }
-  
+
   public String getAppNMHost() {
     return nmHost;
   }
-  
+
   public int getAppNMPort() {
     return nmPort;
   }
-  
+
   public int getAppNMHttpPort() {
     return nmHttpPort;
   }
@@ -680,15 +680,15 @@ public class DAGAppMaster extends CompositeService {
   public DAGAppMasterState getState() {
     return state;
   }
-  
+
   public List<String> getDiagnostics() {
     return dag.getDiagnostics();
   }
-  
+
   public float getProgress() {
     return dag.getProgress();
   }
-  
+
   void setStateOnDAGCompletion() {
     DAGAppMasterState oldState = state;
     if(state == DAGAppMasterState.RUNNING) {
@@ -709,7 +709,7 @@ public class DAGAppMaster extends CompositeService {
     }
     LOG.info("On DAG completion. Old state: " + oldState + " new state: " + state);
   }
-  
+
   class DAGClientHandler implements DAGClient {
 
     @Override
@@ -718,22 +718,22 @@ public class DAGAppMaster extends CompositeService {
     }
 
     @Override
-    public DAGStatus getDAGStatus(String dagIdStr) 
+    public DAGStatus getDAGStatus(String dagIdStr)
                                       throws IOException, TezRemoteException {
       return getDAG(dagIdStr).getDAGStatus();
     }
 
     @Override
-    public VertexStatus getVertexStatus(String dagIdStr, String vertexName) 
+    public VertexStatus getVertexStatus(String dagIdStr, String vertexName)
         throws IOException, TezRemoteException{
       VertexStatus status = getDAG(dagIdStr).getVertexStatus(vertexName);
       if(status == null) {
         throw new TezRemoteException("Unknown vertexName: " + vertexName);
       }
-      
+
       return status;
     }
-    
+
     DAG getDAG(String dagIdStr) throws IOException, TezRemoteException {
       TezDAGID dagId = TezDAGID.fromString(dagIdStr);
       if(dagId == null) {
@@ -759,11 +759,11 @@ public class DAGAppMaster extends CompositeService {
       this.conf = config;
     }
 
-    @Override 
+    @Override
     public DAGAppMaster getAppMaster() {
       return DAGAppMaster.this;
     }
-    
+
     @Override
     public ApplicationAttemptId getApplicationAttemptId() {
       return appAttemptID;
@@ -859,7 +859,7 @@ public class DAGAppMaster extends CompositeService {
   public void start() {
 
     this.state = DAGAppMasterState.RUNNING;
-    
+
     // TODO Recovery
     // Pull completedTasks etc from recovery
     /*
@@ -868,9 +868,9 @@ public class DAGAppMaster extends CompositeService {
       amInfos = recoveryServ.getAMInfos();
     }
     */
-    
+
     // /////////////////// Create the job itself.
-    dag = createDAG(jobPlan);
+    dag = createDAG(dagPlan);
 
     // End of creating the job.
 
@@ -893,7 +893,7 @@ public class DAGAppMaster extends CompositeService {
         startTime, dagsStartTime, appSubmitTime);
     dispatcher.getEventHandler().handle(
         new DAGHistoryEvent(this.dagId, startEvent));
-    
+
     // All components have started, start the job.
     startDags();
   }
@@ -922,7 +922,7 @@ public class DAGAppMaster extends CompositeService {
       ((EventHandler<DAGEvent>)context.getDAG()).handle(event);
     }
   }
-  
+
   private class TaskEventDispatcher implements EventHandler<TaskEvent> {
     @SuppressWarnings("unchecked")
     @Override
@@ -959,7 +959,7 @@ public class DAGAppMaster extends CompositeService {
       ((EventHandler<VertexEvent>) vertex).handle(event);
     }
   }
-  
+
   private static void validateInputParam(String value, String param)
       throws IOException {
     if (value == null) {
@@ -998,22 +998,22 @@ public class DAGAppMaster extends CompositeService {
       // TODO change this once the client is ready.
       String type;
       TezConfiguration conf = new TezConfiguration(new YarnConfiguration());
-      
-      DAGPlan jobPlan = null;
+
+      DAGPlan dagPlan = null;
       if (cliParser.hasOption(OPT_PREDEFINED)) {
         LOG.info("Running with PreDefined configuration");
         type = cliParser.getOptionValue(OPT_PREDEFINED, "mr");
         LOG.info("Running job type: " + type);
 
         if (type.equals("mr")) {
-          jobPlan = MRRExampleHelper.createDAGConfigurationForMR();
+          dagPlan = MRRExampleHelper.createDAGConfigurationForMR();
         } else if (type.equals("mrr")) {
-          jobPlan = MRRExampleHelper.createDAGConfigurationForMRR();
+          dagPlan = MRRExampleHelper.createDAGConfigurationForMRR();
         }
-      } 
+      }
       else {
         // Read the protobuf DAG
-        DAGPlan.Builder dagPlanBuilder = DAGPlan.newBuilder(); 
+        DAGPlan.Builder dagPlanBuilder = DAGPlan.newBuilder();
         FileInputStream dagPBBinaryStream = null;
         try {
           dagPBBinaryStream = new FileInputStream(TezConfiguration.DAG_AM_PLAN_PB_BINARY);
@@ -1021,17 +1021,19 @@ public class DAGAppMaster extends CompositeService {
         }
         finally {
           if(dagPBBinaryStream != null){
-            dagPBBinaryStream.close();  
+            dagPBBinaryStream.close();
           }
         }
 
-        jobPlan = dagPlanBuilder.build();
+        dagPlan = dagPlanBuilder.build();
       }
 
-      LOG.info("XXXX Running a DAG with "
-          + jobPlan.getVertexCount() + " vertices ");
-      for (VertexPlan v : jobPlan.getVertexList()) {
-        LOG.info("XXXX DAG has vertex " + v.getName());
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Running a DAG with "
+            + dagPlan.getVertexCount() + " vertices ");
+        for (VertexPlan v : dagPlan.getVertexList()) {
+          LOG.debug("DAG has vertex " + v.getName());
+        }
       }
 
       String jobUserName = System
@@ -1041,8 +1043,8 @@ public class DAGAppMaster extends CompositeService {
       // SIGTERM I have a chance to write out the job history. I'll be closing
       // the objects myself.
       conf.setBoolean("fs.automatic.close", false);
-      
-      Map<String, String> config = DagTypeConverters.createSettingsMapFromDAGPlan(jobPlan.getJobSettingList());
+
+      Map<String, String> config = DagTypeConverters.createSettingsMapFromDAGPlan(dagPlan.getJobSettingList());
       for(Entry<String, String> entry : config.entrySet()) {
         conf.set(entry.getKey(), entry.getValue());
       }
@@ -1050,7 +1052,7 @@ public class DAGAppMaster extends CompositeService {
       DAGAppMaster appMaster =
           new DAGAppMaster(applicationAttemptId, containerId, nodeHostString,
               Integer.parseInt(nodePortString),
-              Integer.parseInt(nodeHttpPortString), appSubmitTime, jobPlan);
+              Integer.parseInt(nodeHttpPortString), appSubmitTime, dagPlan);
       ShutdownHookManager.get().addShutdownHook(
         new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
 
@@ -1117,7 +1119,7 @@ public class DAGAppMaster extends CompositeService {
       }
     });
   }
-  
+
   @SuppressWarnings("unchecked")
   private void sendEvent(Event<?> event) {
     dispatcher.getEventHandler().handle(event);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 7600db4..708b33e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -5,9 +5,9 @@
  * licenses this file to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
@@ -67,7 +67,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
 
   private static final ContainerTask TASK_FOR_INVALID_JVM = new ContainerTask(
       null, true);
-  
+
   private static ProceedToCompletionResponse COMPLETION_RESPONSE_NO_WAIT =
       new ProceedToCompletionResponse(true, true);
 
@@ -86,10 +86,10 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
   // TODO Use this to figure out whether an incoming ping is valid.
   private ConcurrentMap<TezTaskAttemptID, ContainerId> attemptToContainerIdMap =
       new ConcurrentHashMap<TezTaskAttemptID, ContainerId>();
-  
+
   private Set<ContainerId> registeredContainers = Collections
       .newSetFromMap(new ConcurrentHashMap<ContainerId, Boolean>());
-  
+
   public TaskAttemptListenerImpTezDag(AppContext context,
       TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
       JobTokenSecretManager jobTokenSecretManager) {
@@ -118,10 +118,10 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
               conf.getInt(TezConfiguration.DAG_AM_TASK_LISTENER_THREAD_COUNT,
                   TezConfiguration.DAG_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT))
           .setSecretManager(jobTokenSecretManager).build();
-      
+
       // Enable service authorization?
       if (conf.getBoolean(
-          CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
+          CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
           false)) {
         refreshServiceAcls(conf, new MRAMPolicyProvider());
       }
@@ -133,7 +133,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
     }
   }
 
-  void refreshServiceAcls(Configuration configuration, 
+  void refreshServiceAcls(Configuration configuration,
       PolicyProvider policyProvider) {
     this.server.refreshServiceAcl(configuration, policyProvider);
   }
@@ -153,7 +153,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
   public InetSocketAddress getAddress() {
     return address;
   }
-  
+
   @Override
   public long getProtocolVersion(String protocol, long clientVersion)
       throws IOException {
@@ -171,13 +171,13 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
   public TezTaskDependencyCompletionEventsUpdate getDependentTasksCompletionEvents(
       int fromEventIdx, int maxEvents,
       TezTaskAttemptID taskAttemptID) {
-    
+
     LOG.info("Dependency Completion Events request from " + taskAttemptID
         + ". fromEventID " + fromEventIdx + " maxEvents " + maxEvents);
 
     // TODO: shouldReset is never used. See TT. Ask for Removal.
     boolean shouldReset = false;
-    TezDependentTaskCompletionEvent[] events = 
+    TezDependentTaskCompletionEvent[] events =
         context.getDAG().
             getVertex(taskAttemptID.getTaskID().getVertexID()).
                 getTaskAttemptCompletionEvents(fromEventIdx, maxEvents);
@@ -234,22 +234,28 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
         }
       }
     }
-    LOG.info("DEBUG: getTask returning task: " + task);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("getTask returning task: " + task);
+    }
     return task;
   }
 
   @Override
   public boolean statusUpdate(TezTaskAttemptID taskAttemptId,
       TezTaskStatus taskStatus) throws IOException, InterruptedException {
-    LOG.info("DEBUG: " + "Status update from: " + taskAttemptId);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Status update from: " + taskAttemptId);
+    }
     taskHeartbeatHandler.progressing(taskAttemptId);
     pingContainerHeartbeatHandler(taskAttemptId);
     TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatus();
     taskAttemptStatus.id = taskAttemptId;
     // Task sends the updated progress to the TT.
     taskAttemptStatus.progress = taskStatus.getProgress();
-    LOG.info("DEBUG: " + "Progress of TaskAttempt " + taskAttemptId + " is : "
-        + taskStatus.getProgress());
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Progress of TaskAttempt " + taskAttemptId + " is : "
+          + taskStatus.getProgress());
+    }
 
     // Task sends the updated state-string to the TT.
     taskAttemptStatus.stateString = taskStatus.getStateString();
@@ -267,7 +273,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
     // that is the primary storage format inside the AM to avoid multiple
     // conversions and unnecessary heap usage.
     taskAttemptStatus.counters = taskStatus.getCounters();
-    
+
 
     // Map Finish time set by the task (map only)
     // TODO CLEANMRXAM - maybe differentiate between map / reduce / types
@@ -366,7 +372,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
   /**
    * TaskAttempt is reporting that it is in commit_pending and it is waiting for
    * the commit Response
-   * 
+   *
    * <br/>
    * Commit it a two-phased protocol. First the attempt informs the
    * ApplicationMaster that it is
@@ -386,14 +392,14 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
     //Ignorable TaskStatus? - since a task will send a LastStatusUpdate
     context.getEventHandler().handle(
         new TaskAttemptEvent(
-            taskAttemptId, 
+            taskAttemptId,
             TaskAttemptEventType.TA_COMMIT_PENDING)
         );
   }
 
   /**
    * Child checking whether it can commit.
-   * 
+   *
    * <br/>
    * Commit is a two-phased protocol. First the attempt informs the
    * ApplicationMaster that it is
@@ -411,7 +417,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
     pingContainerHeartbeatHandler(taskAttemptId);
 
     DAG job = context.getDAG();
-    Task task = 
+    Task task =
         job.getVertex(taskAttemptId.getTaskID().getVertexID()).
             getTask(taskAttemptId.getTaskID());
     return task.canCommit(taskAttemptId);
@@ -459,16 +465,16 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
   @Override
   public ProceedToCompletionResponse
       proceedToCompletion(TezTaskAttemptID taskAttemptId) throws IOException {
-    
+
     // The async nature of the processing combined with the 1 second interval
     // between polls (MRTask) implies tasks end up wasting upto 1 second doing
     // nothing. Similarly for CA_COMMIT.
-    
+
     DAG job = context.getDAG();
-    Task task = 
+    Task task =
         job.getVertex(taskAttemptId.getTaskID().getVertexID()).
             getTask(taskAttemptId.getTaskID());
-    
+
     // TODO In-Memory Shuffle
     /*
     if (task.needsWaitAfterOutputConsumable()) {
@@ -503,7 +509,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
     */
     return COMPLETION_RESPONSE_NO_WAIT;
   }
-  
+
 
   @Override
   public void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
@@ -539,11 +545,11 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
     }
     registeredContainers.remove(containerId);
   }
-  
+
   private void pingContainerHeartbeatHandler(ContainerId containerId) {
     containerHeartbeatHandler.pinged(containerId);
   }
-  
+
   private void pingContainerHeartbeatHandler(TezTaskAttemptID taskAttemptId) {
     ContainerId containerId = attemptToContainerIdMap.get(taskAttemptId);
     if (containerId != null) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index d2395ad..de43db0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -558,7 +558,9 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   protected void startRootVertices() {
     for (Vertex v : vertices.values()) {
       if (v.getInputVerticesCount() == 0) {
-        LOG.info("DEBUG: Starting root vertex " + v.getName());
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Starting root vertex " + v.getName());
+        }
         eventHandler.handle(new VertexEvent(v.getVertexId(),
             VertexEventType.V_START));
       }
@@ -577,12 +579,10 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
    * The only entry point to change the DAG.
    */
   public void handle(DAGEvent event) {
-    LOG.info("DEBUG: Processing DAGEvent " + event.getDAGId() + " of type "
-        + event.getType() + " while in state " + getInternalState()
-        + ". Event: " + event);
     if (LOG.isDebugEnabled()) {
       LOG.debug("Processing DAGEvent " + event.getDAGId() + " of type "
-          + event.getType() + " while in state " + getInternalState());
+          + event.getType() + " while in state " + getInternalState()
+          + ". Event: " + event);
     }
     try {
       writeLock.lock();
@@ -658,13 +658,14 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   }
 
   static DAGState checkJobForCompletion(DAGImpl dag) {
-
-    LOG.info("ZZZZ: Checking dag completion"
-        + ", numCompletedVertices=" + dag.numCompletedVertices
-        + ", numSuccessfulVertices=" + dag.numSuccessfulVertices
-        + ", numFailedVertices=" + dag.numFailedVertices
-        + ", numKilledVertices=" + dag.numKilledVertices
-        + ", numVertices=" + dag.numVertices);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Checking dag completion"
+          + ", numCompletedVertices=" + dag.numCompletedVertices
+          + ", numSuccessfulVertices=" + dag.numSuccessfulVertices
+          + ", numFailedVertices=" + dag.numFailedVertices
+          + ", numKilledVertices=" + dag.numKilledVertices
+          + ", numVertices=" + dag.numVertices);
+    }
 
     if (dag.numFailedVertices > 0) {
       dag.setFinishTime();
@@ -870,7 +871,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
         for (Vertex v : dag.vertices.values()) {
           parseVertexEdges(dag, edgePlans, v);
         }
-        
+
         assignDAGScheduler(dag);
 
         // TODO Metrics
@@ -887,7 +888,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
         return dag.finished(DAGState.FAILED);
       }
     }
-    
+
     private void assignDAGScheduler(DAGImpl dag) {
       boolean isMRR = true;
       for(Vertex vertex : dag.vertices.values()) {
@@ -895,15 +896,15 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
         if(outVertices == null || outVertices.isEmpty()) {
           continue;
         }
-        if(outVertices.size() > 1 || 
-           outVertices.values().iterator().next().getConnectionPattern() != 
+        if(outVertices.size() > 1 ||
+           outVertices.values().iterator().next().getConnectionPattern() !=
            EdgeProperty.ConnectionPattern.BIPARTITE) {
           // more than 1 output OR single output is not bipartite
           isMRR = false;
           break;
-        }          
+        }
       }
-      
+
       if(isMRR) {
         LOG.info("Using MRR dag scheduler");
         dag.dagScheduler = new DAGSchedulerMRR(dag, dag.eventHandler);
@@ -1110,10 +1111,11 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
     public DAGState transition(DAGImpl job, DAGEvent event) {
 
       DAGEventVertexCompleted vertexEvent = (DAGEventVertexCompleted) event;
-      LOG.info("DEBUG: Received a vertex completion event"
-          + ", vertexId=" + vertexEvent.getVertexId()
-          + ", vertexState=" + vertexEvent.getVertexState());
-
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Received a vertex completion event"
+            + ", vertexId=" + vertexEvent.getVertexId()
+            + ", vertexState=" + vertexEvent.getVertexState());
+      }
       Vertex vertex = job.vertices.get(vertexEvent.getVertexId());
       job.numCompletedVertices++;
       if (vertexEvent.getVertexState() == VertexState.SUCCEEDED) {
@@ -1124,12 +1126,14 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
         vertexKilled(job, vertex);
       }
 
-      LOG.info("ZZZZ: Vertex completed."
-          + ", numCompletedVertices=" + job.numCompletedVertices
-          + ", numSuccessfulVertices=" + job.numSuccessfulVertices
-          + ", numFailedVertices=" + job.numFailedVertices
-          + ", numKilledVertices=" + job.numKilledVertices
-          + ", numVertices=" + job.numVertices);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Vertex completed."
+            + ", numCompletedVertices=" + job.numCompletedVertices
+            + ", numSuccessfulVertices=" + job.numSuccessfulVertices
+            + ", numFailedVertices=" + job.numFailedVertices
+            + ", numKilledVertices=" + job.numKilledVertices
+            + ", numVertices=" + job.numVertices);
+      }
 
       job.dagScheduler.vertexCompleted(vertex);
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index c14ba8a..94e5161 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -109,7 +109,7 @@ public class TaskAttemptImpl implements TaskAttempt,
   private static final Log LOG = LogFactory.getLog(TaskAttemptImpl.class);
   private static final String LINE_SEPARATOR = System
       .getProperty("line.separator");
-  
+
   static final TezCounters EMPTY_COUNTERS = new TezCounters();
 
   protected final TezConfiguration conf;
@@ -147,22 +147,22 @@ public class TaskAttemptImpl implements TaskAttempt,
   protected final boolean isRescheduled;
 
   protected String processorName;
-  
+
   protected static final FailedTransitionHelper FAILED_HELPER =
       new FailedTransitionHelper();
-      
+
   protected static final KilledTransitionHelper KILLED_HELPER =
       new KilledTransitionHelper();
 
   private static SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent>
       DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION =
           new DiagnosticInformationUpdater();
-  
+
   private static SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent>
       STATUS_UPDATER = new StatusUpdaterTransition();
 
   private final StateMachine<TaskAttemptStateInternal, TaskAttemptEventType, TaskAttemptEvent> stateMachine;
-  
+
   private static StateMachineFactory
   <TaskAttemptImpl, TaskAttemptStateInternal, TaskAttemptEventType, TaskAttemptEvent>
   stateMachineFactory
@@ -174,7 +174,7 @@ public class TaskAttemptImpl implements TaskAttempt,
         .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.NEW, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
         .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_FAIL_REQUEST, new TerminateTransition(FAILED_HELPER))
         .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_KILL_REQUEST, new TerminateTransition(KILLED_HELPER))
-        
+
         .addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_STARTED_REMOTELY, new StartedTransition())
         .addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.START_WAIT, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
         .addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptEventType.TA_FAIL_REQUEST, new TerminatedBeforeRunningTransition(FAILED_HELPER))
@@ -182,7 +182,7 @@ public class TaskAttemptImpl implements TaskAttempt,
         .addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptEventType.TA_NODE_FAILED, new NodeFailedBeforeRunningTransition())
         .addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptEventType.TA_CONTAINER_TERMINATING, new ContainerTerminatingBeforeRunningTransition())
         .addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATED, new ContainerCompletedBeforeRunningTransition())
-        
+
         .addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_STATUS_UPDATE, STATUS_UPDATER)
         .addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
         .addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE, new OutputConsumableTransition()) //Optional, may not come in for all tasks.
@@ -195,7 +195,7 @@ public class TaskAttemptImpl implements TaskAttempt,
         .addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptEventType.TA_NODE_FAILED, new TerminatedWhileRunningTransition(KILLED_HELPER))
         .addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptEventType.TA_CONTAINER_TERMINATING, new TerminatedWhileRunningTransition(FAILED_HELPER))
         .addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATED, new ContainerCompletedWhileRunningTransition())
-        
+
         .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_STATUS_UPDATE, STATUS_UPDATER)
         .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
         .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE) // Stuck RPC. The client retries in a loop.
@@ -227,17 +227,17 @@ public class TaskAttemptImpl implements TaskAttempt,
         .addTransition(TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_CONTAINER_TERMINATED, new ContainerCompletedWhileTerminating())
         .addTransition(TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
         .addTransition(TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptStateInternal.KILL_IN_PROGRESS, EnumSet.of(TaskAttemptEventType.TA_STARTED_REMOTELY, TaskAttemptEventType.TA_STATUS_UPDATE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED, TaskAttemptEventType.TA_TIMED_OUT, TaskAttemptEventType.TA_FAIL_REQUEST, TaskAttemptEventType.TA_KILL_REQUEST, TaskAttemptEventType.TA_NODE_FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATING, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES))
-        
+
         .addTransition(TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATED, new ContainerCompletedWhileTerminating())
         .addTransition(TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
         .addTransition(TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptStateInternal.FAIL_IN_PROGRESS, EnumSet.of(TaskAttemptEventType.TA_STARTED_REMOTELY, TaskAttemptEventType.TA_STATUS_UPDATE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED, TaskAttemptEventType.TA_TIMED_OUT, TaskAttemptEventType.TA_FAIL_REQUEST, TaskAttemptEventType.TA_KILL_REQUEST, TaskAttemptEventType.TA_NODE_FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATING, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES))
-        
+
         .addTransition(TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
         .addTransition(TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.KILLED, EnumSet.of(TaskAttemptEventType.TA_STARTED_REMOTELY, TaskAttemptEventType.TA_STATUS_UPDATE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED, TaskAttemptEventType.TA_TIMED_OUT, TaskAttemptEventType.TA_FAIL_REQUEST, TaskAttemptEventType.TA_KILL_REQUEST, TaskAttemptEventType.TA_NODE_FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATING, TaskAttemptEventType.TA_CONTAINER_TERMINATED, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES))
 
         .addTransition(TaskAttemptStateInternal.FAILED, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
         .addTransition(TaskAttemptStateInternal.FAILED, TaskAttemptStateInternal.FAILED, EnumSet.of(TaskAttemptEventType.TA_STARTED_REMOTELY, TaskAttemptEventType.TA_STATUS_UPDATE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED, TaskAttemptEventType.TA_TIMED_OUT, TaskAttemptEventType.TA_FAIL_REQUEST, TaskAttemptEventType.TA_KILL_REQUEST, TaskAttemptEventType.TA_NODE_FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATING, TaskAttemptEventType.TA_CONTAINER_TERMINATED, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES))
-        
+
         // How will duplicate history events be handled ?
         // TODO Maybe consider not failing REDUCE tasks in this case. Also, MAP_TASKS in case there's only one phase in the job.
         .addTransition(TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.SUCCEEDED, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
@@ -245,21 +245,21 @@ public class TaskAttemptImpl implements TaskAttempt,
         .addTransition(TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_NODE_FAILED, new TerminatedAfterSuccessTransition(KILLED_HELPER))
         .addTransition(TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES, new TerminatedAfterSuccessTransition(FAILED_HELPER))
         .addTransition(TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.SUCCEEDED, EnumSet.of(TaskAttemptEventType.TA_TIMED_OUT, TaskAttemptEventType.TA_FAIL_REQUEST, TaskAttemptEventType.TA_CONTAINER_TERMINATING, TaskAttemptEventType.TA_CONTAINER_TERMINATED))
-        
-        
+
+
         .installTopology();
-  
+
 
   // TODO Remove TaskAttemptListener from the constructor.
   @SuppressWarnings("rawtypes")
   public TaskAttemptImpl(TezTaskID taskId, int attemptNumber, EventHandler eventHandler,
-      TaskAttemptListener tal, int partition, 
+      TaskAttemptListener tal, int partition,
       TezConfiguration conf,
       Token<JobTokenIdentifier> jobToken, Credentials credentials, Clock clock,
       TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
       String processorName, TaskLocationHint locationHint,
       Resource resource, Map<String, LocalResource> localResources,
-      Map<String, String> environment, 
+      Map<String, String> environment,
       String javaOpts, boolean isRescheduled) {
     ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
     this.readLock = rwLock.readLock();
@@ -286,34 +286,34 @@ public class TaskAttemptImpl implements TaskAttempt,
     this.javaOpts = javaOpts;
     this.isRescheduled = isRescheduled;
   }
-  
+
 
   @Override
   public TezTaskAttemptID getID() {
     return attemptId;
   }
-  
+
   @Override
   public TezVertexID getVertexID() {
     return attemptId.getTaskID().getVertexID();
   }
-  
+
   @Override
   public TezDAGID getDAGID() {
     return getVertexID().getDAGId();
   }
-  
+
   TezTaskContext createRemoteTask() {
     Vertex vertex = getTask().getVertex();
     DAG dag = vertex.getDAG();
 
     // TODO  TEZ-50 user and jobname
-    return new TezEngineTaskContext(getID(), dag.getUserName(), 
+    return new TezEngineTaskContext(getID(), dag.getUserName(),
         dag.getName(), getTask()
         .getVertex().getName(), processorName,
         vertex.getInputSpecList(), vertex.getOutputSpecList());
   }
-  
+
   @Override
   public TaskAttemptReport getReport() {
     TaskAttemptReport result = Records.newRecord(TaskAttemptReport.class);
@@ -322,7 +322,7 @@ public class TaskAttemptImpl implements TaskAttempt,
       result.setTaskAttemptId(attemptId);
       //take the LOCAL state of attempt
       //DO NOT take from reportedStatus
-      
+
       result.setTaskAttemptState(getState());
       result.setProgress(reportedStatus.progress);
       result.setStartTime(launchTime);
@@ -522,11 +522,9 @@ public class TaskAttemptImpl implements TaskAttempt,
   public void handle(TaskAttemptEvent event) {
     if (LOG.isDebugEnabled()) {
       LOG.debug("Processing TaskAttemptEvent " + event.getTaskAttemptID()
-          + " of type " + event.getType());
+          + " of type " + event.getType() + " while in state "
+          + getInternalState() + ". Event: " + event);
     }
-    LOG.info("DEBUG: Processing TaskAttemptEvent " + event.getTaskAttemptID()
-        + " of type " + event.getType() + " while in state "
-        + getInternalState() + ". Event: " + event);
     writeLock.lock();
     try {
       final TaskAttemptStateInternal oldState = getInternalState();
@@ -536,8 +534,8 @@ public class TaskAttemptImpl implements TaskAttempt,
         LOG.error("Can't handle this event at current state for "
             + this.attemptId, e);
         eventHandler.handle(new DAGEventDiagnosticsUpdate(
-            this.attemptId.getTaskID().getVertexID().getDAGId(), 
-            "Invalid event " + event.getType() + 
+            this.attemptId.getTaskID().getVertexID().getDAGId(),
+            "Invalid event " + event.getType() +
             " on TaskAttempt " + this.attemptId));
         eventHandler.handle(
             new DAGEvent(
@@ -546,7 +544,7 @@ public class TaskAttemptImpl implements TaskAttempt,
             );
       }
       if (oldState != getInternalState()) {
-          LOG.info(attemptId + " TaskAttempt Transitioned from " 
+          LOG.info(attemptId + " TaskAttempt Transitioned from "
            + oldState + " to "
            + getInternalState());
       }
@@ -554,7 +552,7 @@ public class TaskAttemptImpl implements TaskAttempt,
       writeLock.unlock();
     }
   }
-  
+
   @VisibleForTesting
   public TaskAttemptStateInternal getInternalState() {
     readLock.lock();
@@ -562,7 +560,7 @@ public class TaskAttemptImpl implements TaskAttempt,
       return stateMachine.getCurrentState();
     } finally {
       readLock.unlock();
-    }    
+    }
   }
 
   private static TaskAttemptState getExternalState(
@@ -590,7 +588,7 @@ public class TaskAttemptImpl implements TaskAttempt,
           + smState);
     }
   }
-  
+
   @Override
   public boolean getIsRescheduled() {
     return isRescheduled;
@@ -612,7 +610,7 @@ public class TaskAttemptImpl implements TaskAttempt,
   // TOOD Merge some of these JobCounter events.
   private static DAGEventCounterUpdate createJobCounterUpdateEventTALaunched(
       TaskAttemptImpl ta) {
-    DAGEventCounterUpdate jce = 
+    DAGEventCounterUpdate jce =
         new DAGEventCounterUpdate(
             ta.getDAGID()
             );
@@ -622,8 +620,8 @@ public class TaskAttemptImpl implements TaskAttempt,
 
   private static DAGEventCounterUpdate createJobCounterUpdateEventSlotMillis(
       TaskAttemptImpl ta) {
-    DAGEventCounterUpdate jce = 
-        new DAGEventCounterUpdate(            
+    DAGEventCounterUpdate jce =
+        new DAGEventCounterUpdate(
             ta.getDAGID()
             );
 
@@ -635,7 +633,7 @@ public class TaskAttemptImpl implements TaskAttempt,
   private static DAGEventCounterUpdate createJobCounterUpdateEventTATerminated(
       TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted,
       TaskAttemptStateInternal taState) {
-    DAGEventCounterUpdate jce = 
+    DAGEventCounterUpdate jce =
         new DAGEventCounterUpdate(
             taskAttempt.getDAGID());
 
@@ -687,16 +685,16 @@ public class TaskAttemptImpl implements TaskAttempt,
         taskAttempt.finishTime,
         taskAttempt.containerNodeId == null ? "UNKNOWN"
             : taskAttempt.containerNodeId.getHost(),
-        taskAttempt.containerNodeId == null ? -1 
-            : taskAttempt.containerNodeId.getPort(),    
-        taskAttempt.nodeRackName == null ? "UNKNOWN" 
+        taskAttempt.containerNodeId == null ? -1
+            : taskAttempt.containerNodeId.getPort(),
+        taskAttempt.nodeRackName == null ? "UNKNOWN"
             : taskAttempt.nodeRackName,
         StringUtils.join(
             LINE_SEPARATOR, taskAttempt.getDiagnostics()), taskAttempt
             .getProgressSplitBlock().burst());
     return tauce;
   }
-  
+
   // TODO Incorporate MAPREDUCE-4838
   private JobHistoryEvent createTaskAttemptStartedEvent() {
     TaskAttemptStartedEvent tase = new TaskAttemptStartedEvent(
@@ -707,7 +705,7 @@ public class TaskAttemptImpl implements TaskAttempt,
 
   }
   */
-  
+
 //  private WrappedProgressSplitsBlock getProgressSplitBlock() {
 //    return null;
 //    // TODO
@@ -725,7 +723,7 @@ public class TaskAttemptImpl implements TaskAttempt,
 //    }
 //    */
 //  }
-  
+
   private void updateProgressSplits() {
 //    double newProgress = reportedStatus.progress;
 //    newProgress = Math.max(Math.min(newProgress, 1.0D), 0.0D);
@@ -737,7 +735,7 @@ public class TaskAttemptImpl implements TaskAttempt,
 //    if (splitsBlock != null) {
 //      long now = clock.getTime();
 //      long start = getLaunchTime();
-//      
+//
 //      if (start == 0)
 //        return;
 //
@@ -767,7 +765,7 @@ public class TaskAttemptImpl implements TaskAttempt,
 //      }
 //    }
   }
-  
+
 //  private void maybeSendSpeculatorContainerRequired() {
 //    if (!speculatorContainerRequestSent) {
 //      sendEvent(new SpeculatorEvent(getID().getTaskID(), +1));
@@ -781,10 +779,10 @@ public class TaskAttemptImpl implements TaskAttempt,
 //      speculatorContainerRequestSent = false;
 //    }
 //  }
-  
+
   private void sendTaskAttemptCleanupEvent() {
-//    TaskAttemptContext taContext = 
-//        new TaskAttemptContextImpl(this.conf, 
+//    TaskAttemptContext taContext =
+//        new TaskAttemptContextImpl(this.conf,
 //            TezMRTypeConverter.fromTez(this.attemptId));
 //    sendEvent(new TaskCleanupEvent(this.attemptId, this.committer, taContext));
   }
@@ -825,7 +823,7 @@ public class TaskAttemptImpl implements TaskAttempt,
   protected void logJobHistoryAttemptFinishedEvent(TaskAttemptStateInternal state) {
     //Log finished events only if an attempt started.
     if (getLaunchTime() == 0) return;
-    
+
     TaskAttemptFinishedEvent finishEvt = new TaskAttemptFinishedEvent(
         attemptId, getTask().getVertex().getName(),
         getFinishTime(), TaskAttemptState.SUCCEEDED, "",
@@ -901,7 +899,7 @@ public class TaskAttemptImpl implements TaskAttempt,
               ta.localResources, remoteTaskContext, ta,
               ta.credentials, ta.jobToken, hostArray,
               rackArray,
-              scheduleEvent.getPriority(), ta.environment, //ta.javaOpts, 
+              scheduleEvent.getPriority(), ta.environment, //ta.javaOpts,
               ta.conf);
       ta.sendEvent(launchRequestEvent);
     }
@@ -1269,7 +1267,7 @@ public class TaskAttemptImpl implements TaskAttempt,
       return TaskEventType.T_ATTEMPT_KILLED;
     }
   }
-  
+
   @Override
   public String toString() {
     return getID().toString();
@@ -1284,10 +1282,10 @@ public class TaskAttemptImpl implements TaskAttempt,
   public Map<String, String> getEnvironment() {
     return this.environment;
   }
-  
+
   @Override
   public String getJavaOpts() {
-	return this.javaOpts;  
+	return this.javaOpts;
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 575f32c..f295caf 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -107,68 +107,68 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
   protected Resource taskResource;
   protected Map<String, LocalResource> localResources;
   protected Map<String, String> environment;
-  
+
   // counts the number of attempts that are either running or in a state where
   //  they will come to be running when they get a Container
   private int numberUncompletedAttempts = 0;
 
   private boolean historyTaskStartGenerated = false;
-  
-  private static final SingleArcTransition<TaskImpl, TaskEvent> 
+
+  private static final SingleArcTransition<TaskImpl, TaskEvent>
      ATTEMPT_KILLED_TRANSITION = new AttemptKilledTransition();
-  private static final SingleArcTransition<TaskImpl, TaskEvent> 
+  private static final SingleArcTransition<TaskImpl, TaskEvent>
      KILL_TRANSITION = new KillTransition();
 
   private static final StateMachineFactory
-               <TaskImpl, TaskStateInternal, TaskEventType, TaskEvent> 
-            stateMachineFactory 
+               <TaskImpl, TaskStateInternal, TaskEventType, TaskEvent>
+            stateMachineFactory
            = new StateMachineFactory<TaskImpl, TaskStateInternal, TaskEventType, TaskEvent>
                (TaskStateInternal.NEW)
 
     // define the state machine of Task
 
     // Transitions from NEW state
-    .addTransition(TaskStateInternal.NEW, TaskStateInternal.SCHEDULED, 
+    .addTransition(TaskStateInternal.NEW, TaskStateInternal.SCHEDULED,
         TaskEventType.T_SCHEDULE, new InitialScheduleTransition())
-    .addTransition(TaskStateInternal.NEW, TaskStateInternal.KILLED, 
+    .addTransition(TaskStateInternal.NEW, TaskStateInternal.KILLED,
         TaskEventType.T_KILL, new KillNewTransition())
 
     // Transitions from SCHEDULED state
       //when the first attempt is launched, the task state is set to RUNNING
-     .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.RUNNING, 
+     .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.RUNNING,
          TaskEventType.T_ATTEMPT_LAUNCHED, new LaunchTransition())
-     .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.KILL_WAIT, 
+     .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.KILL_WAIT,
          TaskEventType.T_KILL, KILL_TRANSITION)
-     .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.SCHEDULED, 
+     .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.SCHEDULED,
          TaskEventType.T_ATTEMPT_KILLED, ATTEMPT_KILLED_TRANSITION)
-     .addTransition(TaskStateInternal.SCHEDULED, 
-        EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.FAILED), 
-        TaskEventType.T_ATTEMPT_FAILED, 
+     .addTransition(TaskStateInternal.SCHEDULED,
+        EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.FAILED),
+        TaskEventType.T_ATTEMPT_FAILED,
         new AttemptFailedTransition())
- 
+
     // Transitions from RUNNING state
-    .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, 
+    .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
         TaskEventType.T_ATTEMPT_LAUNCHED) //more attempts may start later
-    // This is an optional event. 
+    // This is an optional event.
     .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
         TaskEventType.T_ATTEMPT_OUTPUT_CONSUMABLE,
         new AttemptProcessingCompleteTransition())
-    .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, 
+    .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
         TaskEventType.T_ATTEMPT_COMMIT_PENDING,
         new AttemptCommitPendingTransition())
     .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
         TaskEventType.T_ADD_SPEC_ATTEMPT, new RedundantScheduleTransition())
-    .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.SUCCEEDED, 
+    .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.SUCCEEDED,
         TaskEventType.T_ATTEMPT_SUCCEEDED,
         new AttemptSucceededTransition())
-    .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, 
+    .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
         TaskEventType.T_ATTEMPT_KILLED,
         ATTEMPT_KILLED_TRANSITION)
-    .addTransition(TaskStateInternal.RUNNING, 
-        EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.FAILED), 
+    .addTransition(TaskStateInternal.RUNNING,
+        EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.FAILED),
         TaskEventType.T_ATTEMPT_FAILED,
         new AttemptFailedTransition())
-    .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.KILL_WAIT, 
+    .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.KILL_WAIT,
         TaskEventType.T_KILL, KILL_TRANSITION)
 
     // Transitions from KILL_WAIT state
@@ -203,7 +203,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
         EnumSet.of(TaskEventType.T_ADD_SPEC_ATTEMPT,
             TaskEventType.T_ATTEMPT_LAUNCHED))
 
-    // Transitions from FAILED state        
+    // Transitions from FAILED state
     .addTransition(TaskStateInternal.FAILED, TaskStateInternal.FAILED,
         EnumSet.of(TaskEventType.T_KILL,
                    TaskEventType.T_ADD_SPEC_ATTEMPT))
@@ -221,7 +221,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
 
   // TODO: Recovery
   /*
-  // By default, the next TaskAttempt number is zero. Changes during recovery  
+  // By default, the next TaskAttempt number is zero. Changes during recovery
   protected int nextAttemptNumber = 0;
   private List<TaskAttemptInfo> taskAttemptsFromPreviousGeneration =
       new ArrayList<TaskAttemptInfo>();
@@ -239,10 +239,10 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
       new RecoverdAttemptsComparator();
 
    */
-  
+
   private TezTaskAttemptID outputConsumableAttempt;
   private boolean outputConsumableAttemptSuccessSent = false;
-  
+
   //should be set to one which comes first
   //saying COMMIT_PENDING
   private TezTaskAttemptID commitAttempt;
@@ -272,10 +272,10 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
       Token<JobTokenIdentifier> jobToken,
       Credentials credentials, Clock clock,
       // TODO Recovery
-      //Map<TezTaskID, TaskInfo> completedTasksFromPreviousRun, 
+      //Map<TezTaskID, TaskInfo> completedTasksFromPreviousRun,
       //int startCount,
       // TODO Metrics
-      //MRAppMetrics metrics, 
+      //MRAppMetrics metrics,
       TaskHeartbeatHandler thh, AppContext appContext,
       String processorName,
       boolean leafVertex, TaskLocationHint locationHint, Resource resource,
@@ -289,7 +289,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     writeLock = readWriteLock.writeLock();
     this.attempts = Collections.emptyMap();
     // TODO TEZ-47 get from conf or API
-    maxAttempts = this.conf.getInt(TezConfiguration.DAG_MAX_TASK_ATTEMPTS, 
+    maxAttempts = this.conf.getInt(TezConfiguration.DAG_MAX_TASK_ATTEMPTS,
                               TezConfiguration.DAG_MAX_TASK_ATTEMPTS_DEFAULT);
     taskId = new TezTaskID(vertexId, partition);
     this.partition = partition;
@@ -360,7 +360,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
       if (attempts.size() <= 1) {
         return attempts;
       }
-      
+
       Map<TezTaskAttemptID, TaskAttempt> result
           = new LinkedHashMap<TezTaskAttemptID, TaskAttempt>();
       result.putAll(attempts);
@@ -385,7 +385,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
   public Vertex getVertex() {
     return appContext.getDAG().getVertex(taskId.getVertexID());
   }
-  
+
   @Override
   public TezTaskID getTaskId() {
     return taskId;
@@ -424,12 +424,12 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
       }
 
       report.setSuccessfulAttempt(successfulAttempt);
-      
+
       for (TaskAttempt att : attempts.values()) {
         String prefix = "AttemptID:" + att.getID() + " Info:";
         for (CharSequence cs : att.getDiagnostics()) {
           report.addDiagnostics(prefix + cs);
-          
+
         }
       }
 
@@ -482,7 +482,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
       return stateMachine.getCurrentState();
     } finally {
       readLock.unlock();
-    }    
+    }
   }
 
   private static TaskState getExternalState(TaskStateInternal smState) {
@@ -490,7 +490,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
       return TaskState.KILLED;
     } else {
       return TaskState.valueOf(smState.name());
-    }    
+    }
   }
 
   //this is always called in read/write lock
@@ -530,7 +530,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     }
     return finishTime;
   }
-  
+
   private TaskStateInternal finished(TaskStateInternal finalState) {
     if (getInternalState() == TaskStateInternal.RUNNING) {
       // TODO Metrics
@@ -552,7 +552,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
       case KILLED:
         continue;
       default:
-      }      
+      }
       if (result == null) {
         result = at; //The first time around
       }
@@ -589,8 +589,8 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
       return false;
     }
   }
-  
-  
+
+
   @Override
   public TezTaskAttemptID getOutputConsumableAttempt() {
     readLock.lock();
@@ -608,7 +608,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
         appContext, processorName, locationHint, taskResource,
         localResources, environment, javaOpts, (failedAttempts>0));
   }
-  
+
   protected TaskAttempt getSuccessfulAttempt() {
     readLock.lock();
     try {
@@ -631,7 +631,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
       case 0:
         attempts = Collections.singletonMap(attempt.getID(), attempt);
         break;
-        
+
       case 1:
         Map<TezTaskAttemptID, TaskAttempt> newAttempts
             = new LinkedHashMap<TezTaskAttemptID, TaskAttempt>(maxAttempts);
@@ -663,17 +663,15 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     // picture in mind
     eventHandler.handle(new DAGEventSchedulerUpdate(
         DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, attempt));
-    
+
   }
 
   @Override
   public void handle(TaskEvent event) {
-    LOG.info("DEBUG: Processing TaskEvent " + event.getTaskID() + " of type "
-        + event.getType() + " while in state " + getInternalState()
-        + ". Event: " + event);
     if (LOG.isDebugEnabled()) {
       LOG.debug("Processing TaskEvent " + event.getTaskID() + " of type "
-          + event.getType());
+          + event.getType() + " while in state " + getInternalState()
+          + ". Event: " + event);
     }
     try {
       writeLock.lock();
@@ -698,7 +696,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
   protected void internalError(TaskEventType type) {
     LOG.error("Invalid event " + type + " on Task " + this.taskId);
     eventHandler.handle(new DAGEventDiagnosticsUpdate(
-        this.taskId.getVertexID().getDAGId(), "Invalid event " + type + 
+        this.taskId.getVertexID().getDAGId(), "Invalid event " + type +
         " on Task " + this.taskId));
     eventHandler.handle(new DAGEvent(this.taskId.getVertexID().getDAGId(),
         DAGEventType.INTERNAL_ERROR));
@@ -719,19 +717,19 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
         }
       }
     }
-    if (attempt.getNodeHttpAddress() != null) {      
-      
+    if (attempt.getNodeHttpAddress() != null) {
+
       String scheme = (encryptedShuffle) ? "https://" : "http://";
       String url = scheme
           + attempt.getNodeHttpAddress().split(":")[0] + ":"
           + attempt.getShufflePort();
-      
-      
-      
+
+
+
       int runTime = 0;
       if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() != 0)
         runTime = (int) (attempt.getFinishTime() - attempt.getLaunchTime());
-      
+
       TezDependentTaskCompletionEvent tce = new TezDependentTaskCompletionEvent(
           -1, attemptId, status, url, runTime);
 
@@ -740,7 +738,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
       eventHandler.handle(new VertexEventTaskAttemptCompleted(tce));
     }
   }
-  
+
   // always called inside a transition, in turn inside the Write Lock
   private void handleTaskAttemptCompletion(TezTaskAttemptID attemptId,
       TezDependentTaskCompletionEvent.Status status) {
@@ -759,7 +757,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
         task.getCounters());
     return tfe;
   }
-  
+
   private static TaskFailedEvent createTaskFailedEvent(TaskImpl task, List<String> diag, TaskStateInternal taskState, TezTaskAttemptID taId) {
     StringBuilder errorSb = new StringBuilder();
     if (diag != null) {
@@ -778,7 +776,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     return taskFailedEvent;
   }
   */
-  
+
   private static void unSucceed(TaskImpl task) {
     task.commitAttempt = null;
     task.successfulAttempt = null;
@@ -801,7 +799,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     this.eventHandler.handle(new DAGHistoryEvent(
         taskId.getVertexID().getDAGId(), startEvt));
   }
-  
+
   protected void logJobHistoryTaskFinishedEvent() {
     // FIXME need to handle getting finish time as this function
     // is called from within a transition
@@ -811,7 +809,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     this.eventHandler.handle(new DAGHistoryEvent(
         taskId.getVertexID().getDAGId(), finishEvt));
   }
-  
+
   protected void logJobHistoryTaskFailedEvent(TaskState finalState) {
     TaskFinishedEvent finishEvt = new TaskFinishedEvent(taskId,
         getVertex().getName(), clock.getTime(), finalState, getCounters());
@@ -859,15 +857,19 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
             TezDependentTaskCompletionEvent.Status.SUCCEEDED);
         task.outputConsumableAttempt = attemptId;
         task.outputConsumableAttemptSuccessSent = true;
-        LOG.info("DEBUG: TezTaskAttemptID: " + attemptId
-            + " set as the OUTPUT_READY attempt");
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("TezTaskAttemptID: " + attemptId
+              + " set as the OUTPUT_READY attempt");
+        }
       } else {
         // Nothing to do. This task will eventually be told to die, or will be
         // killed.
-        LOG.info("DEBUG: TezTaskAttemptID: "
-            + attemptId
-            + " reporting OUTPUT_READY. Will be asked to die since another attempt "
-            + task.outputConsumableAttempt + " already has output ready");
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("TezTaskAttemptID: "
+              + attemptId + " reporting OUTPUT_READY."
+              + " Will be asked to die since another attempt "
+              + task.outputConsumableAttempt + " already has output ready");
+        }
         task.eventHandler.handle(new TaskAttemptEventKillRequest(attemptId,
             "Alternate attemptId already serving output"));
       }
@@ -899,12 +901,12 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     }
   }
 
-  private static class AttemptSucceededTransition 
+  private static class AttemptSucceededTransition
       implements SingleArcTransition<TaskImpl, TaskEvent> {
     @Override
     public void transition(TaskImpl task, TaskEvent event) {
       task.handleTaskAttemptCompletion(
-          ((TaskEventTAUpdate) event).getTaskAttemptID(), 
+          ((TaskEventTAUpdate) event).getTaskAttemptID(),
           TezDependentTaskCompletionEvent.Status.SUCCEEDED);
       task.finishedAttempts++;
       --task.numberUncompletedAttempts;
@@ -937,7 +939,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     @Override
     public void transition(TaskImpl task, TaskEvent event) {
       task.handleTaskAttemptCompletion(
-          ((TaskEventTAUpdate) event).getTaskAttemptID(), 
+          ((TaskEventTAUpdate) event).getTaskAttemptID(),
           TezDependentTaskCompletionEvent.Status.KILLED);
       task.finishedAttempts++;
       --task.numberUncompletedAttempts;
@@ -956,7 +958,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     @Override
     public TaskStateInternal transition(TaskImpl task, TaskEvent event) {
       task.handleTaskAttemptCompletion(
-          ((TaskEventTAUpdate) event).getTaskAttemptID(), 
+          ((TaskEventTAUpdate) event).getTaskAttemptID(),
           TezDependentTaskCompletionEvent.Status.KILLED);
       task.finishedAttempts++;
       // check whether all attempts are finished
@@ -997,7 +999,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
       task.finishedAttempts++;
       if (task.failedAttempts < task.maxAttempts) {
         task.handleTaskAttemptCompletion(
-            ((TaskEventTAUpdate) event).getTaskAttemptID(), 
+            ((TaskEventTAUpdate) event).getTaskAttemptID(),
             TezDependentTaskCompletionEvent.Status.FAILED);
         // we don't need a new event if we already have a spare
         if (--task.numberUncompletedAttempts == 0
@@ -1006,9 +1008,9 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
         }
       } else {
         task.handleTaskAttemptCompletion(
-            ((TaskEventTAUpdate) event).getTaskAttemptID(), 
+            ((TaskEventTAUpdate) event).getTaskAttemptID(),
             TezDependentTaskCompletionEvent.Status.TIPFAILED);
-        
+
         if (task.historyTaskStartGenerated) {
           task.logJobHistoryTaskFailedEvent(TaskState.FAILED);
         } else {
@@ -1046,7 +1048,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
         LOG.error("Unexpected event for task of leaf vertex " + event.getType());
         task.internalError(event.getType());
       }
-      
+
       // tell the job about the rescheduling
       task.eventHandler.handle(
           new VertexEventTaskReschedule(task.taskId));
@@ -1085,16 +1087,16 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
         // tell the job about the rescheduling
         unSucceed(task);
         task.handleTaskAttemptCompletion(
-            attemptId, 
+            attemptId,
             TezDependentTaskCompletionEvent.Status.KILLED);
         task.eventHandler.handle(new VertexEventTaskReschedule(task.taskId));
-        // typically we are here because this map task was run on a bad node and 
+        // typically we are here because this map task was run on a bad node and
         // we want to reschedule it on a different node.
-        // Depending on whether there are previous failed attempts or not this 
+        // Depending on whether there are previous failed attempts or not this
         // can SCHEDULE or RESCHEDULE the container allocate request. If this
         // SCHEDULE's then the dataLocal hosts of this taskAttempt will be used
-        // from the map splitInfo. So the bad node might be sent as a location 
-        // to the RM. But the RM would ignore that just like it would ignore 
+        // from the map splitInfo. So the bad node might be sent as a location
+        // to the RM. But the RM would ignore that just like it would ignore
         // currently pending container requests affinitized to bad nodes.
         task.addAndScheduleAttempt();
         return TaskStateInternal.SCHEDULED;
@@ -1105,11 +1107,11 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     }
   }
 
-  private static class KillNewTransition 
+  private static class KillNewTransition
     implements SingleArcTransition<TaskImpl, TaskEvent> {
     @Override
     public void transition(TaskImpl task, TaskEvent event) {
-      
+
       if (task.historyTaskStartGenerated) {
         task.logJobHistoryTaskFailedEvent(TaskState.KILLED);
       } else {
@@ -1131,7 +1133,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     }
   }
 
-  private static class KillTransition 
+  private static class KillTransition
     implements SingleArcTransition<TaskImpl, TaskEvent> {
     @Override
     public void transition(TaskImpl task, TaskEvent event) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index b06e264..c8ee314 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -599,12 +599,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
    * The only entry point to change the Vertex.
    */
   public void handle(VertexEvent event) {
-    LOG.info("DEBUG: Processing VertexEvent " + event.getVertexId()
-        + " of type " + event.getType() + " while in state "
-        + getInternalState() + ". Event: " + event);
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Processing VertexEvent " + event.getVertexId() + " of type "
-          + event.getType() + " while in state " + getInternalState());
+      LOG.debug("Processing VertexEvent " + event.getVertexId()
+          + " of type " + event.getType() + " while in state "
+          + getInternalState() + ". Event: " + event);
     }
     try {
       writeLock.lock();
@@ -631,9 +629,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     finally {
       writeLock.unlock();
     }
-    LOG.info("DEBUG: Finished processing VertexEvent " + event.getVertexId()
-        + " of type " + event.getType() + " while in state "
-        + getInternalState() + ". Event: " + event);
   }
 
   private VertexState getInternalState() {
@@ -699,12 +694,13 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
 
   static VertexState checkVertexForCompletion(VertexImpl vertex) {
     //check for vertex failure first
-
-    LOG.info("ZZZZ: checking for vertex completion"
-        + ", failedTaskCount=" + vertex.failedTaskCount
-        + ", killedTaskCount=" + vertex.killedTaskCount
-        + ", successfulTaskCount=" + vertex.succeededTaskCount
-        + ", completedTaskCount=" + vertex.completedTaskCount);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Checking for vertex completion"
+          + ", failedTaskCount=" + vertex.failedTaskCount
+          + ", killedTaskCount=" + vertex.killedTaskCount
+          + ", successfulTaskCount=" + vertex.succeededTaskCount
+          + ", completedTaskCount=" + vertex.completedTaskCount);
+    }
 
     if (vertex.failedTaskCount > 0) {
       vertex.setFinishTime();
@@ -1287,8 +1283,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       InputSpec inputSpec = new InputSpec(entry.getKey().getName(),
           entry.getKey().getTotalTasks(),
           entry.getValue().getInputClass());
-      LOG.info("DEBUG: For vertex : " + this.getName()
-          + ", Using InputSpec : " + inputSpec);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("For vertex : " + this.getName()
+            + ", Using InputSpec : " + inputSpec);
+      }
       // TODO DAGAM This should be based on the edge type.
       inputSpecList.add(inputSpec);
     }
@@ -1304,8 +1302,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
         OutputSpec outputSpec = new OutputSpec(entry.getKey().getName(),
             entry.getKey().getTotalTasks(),
             entry.getValue().getOutputClass());
-        LOG.info("DEBUG: For vertex : " + this.getName()
-            + ", Using OutputSpec : " + outputSpec);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("For vertex : " + this.getName()
+              + ", Using OutputSpec : " + outputSpec);
+        }
         // TODO DAGAM This should be based on the edge type.
         outputSpecList.add(outputSpec);
       }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 58e9d50..bab02da 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -59,7 +59,7 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
 public class AMContainerImpl implements AMContainer {
 
   private static final Log LOG = LogFactory.getLog(AMContainerImpl.class);
-  
+
   private final ReadLock readLock;
   private final WriteLock writeLock;
   private final ContainerId containerId;
@@ -75,10 +75,10 @@ public class AMContainerImpl implements AMContainer {
   // TODO Maybe this should be pulled from the TaskAttempt.s
   private final Map<TezTaskAttemptID, TezTaskContext> remoteTaskMap =
       new HashMap<TezTaskAttemptID, TezTaskContext>();
-  
+
   // TODO ?? Convert to list and hash.
-  
-  private int shufflePort; 
+
+  private int shufflePort;
   private long idleTimeBetweenTasks = 0;
   private long lastTaskFinishTime;
 
@@ -92,16 +92,16 @@ public class AMContainerImpl implements AMContainer {
   private TezTaskAttemptID runningAttempt;
   private List<TezTaskAttemptID> failedAssignments;
   private TezTaskAttemptID pullAttempt;
-  
+
   private AMContainerTask noAllocationContainerTask;
-  
+
   private static final AMContainerTask NO_MORE_TASKS = new AMContainerTask(
       true, null);
   private static final AMContainerTask WAIT_TASK = new AMContainerTask(false,
       null);
-  
+
   private boolean inError = false;
-  
+
   private ContainerLaunchContext clc;
 
   // TODO Consider registering with the TAL, instead of the TAL pulling.
@@ -115,8 +115,8 @@ public class AMContainerImpl implements AMContainer {
 
   private final StateMachine<AMContainerState, AMContainerEventType, AMContainerEvent> stateMachine;
   private static final StateMachineFactory
-      <AMContainerImpl, AMContainerState, AMContainerEventType, AMContainerEvent> 
-      stateMachineFactory = 
+      <AMContainerImpl, AMContainerState, AMContainerEventType, AMContainerEvent>
+      stateMachineFactory =
       new StateMachineFactory<AMContainerImpl, AMContainerState, AMContainerEventType, AMContainerEvent>(
       AMContainerState.ALLOCATED)
 
@@ -144,7 +144,7 @@ public class AMContainerImpl implements AMContainer {
         .addTransition(AMContainerState.IDLE, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_TIMED_OUT, new TimedOutAtIdleTransition())
         .addTransition(AMContainerState.IDLE, AMContainerState.STOPPING, AMContainerEventType.C_NODE_FAILED, new NodeFailedAtIdleTransition())
         .addTransition(AMContainerState.IDLE, AMContainerState.STOP_REQUESTED, EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, AMContainerEventType.C_LAUNCHED, AMContainerEventType.C_LAUNCH_FAILED, AMContainerEventType.C_TA_SUCCEEDED, AMContainerEventType.C_NM_STOP_SENT, AMContainerEventType.C_NM_STOP_FAILED), new ErrorAtIdleTransition())
-        
+
         .addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_ASSIGN_TA, new AssignTaskAttemptAtRunningTransition())
         .addTransition(AMContainerState.RUNNING, AMContainerState.RUNNING, AMContainerEventType.C_PULL_TA)
         .addTransition(AMContainerState.RUNNING, AMContainerState.IDLE, AMContainerEventType.C_TA_SUCCEEDED, new TASucceededAtRunningTransition())
@@ -162,7 +162,7 @@ public class AMContainerImpl implements AMContainer {
         .addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.STOPPING, AMContainerEventType.C_NODE_FAILED, new NodeFailedAtNMStopRequestedTransition())
         .addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.STOP_REQUESTED, EnumSet.of(AMContainerEventType.C_LAUNCHED, AMContainerEventType.C_LAUNCH_FAILED, AMContainerEventType.C_TA_SUCCEEDED, AMContainerEventType.C_STOP_REQUEST, AMContainerEventType.C_TIMED_OUT))
         .addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_LAUNCH_REQUEST, new ErrorAtNMStopRequestedTransition())
-        
+
         .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_ASSIGN_TA, new AssignTAAtWindDownTransition())
         .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_PULL_TA, new PullTAAfterStopTransition())
         // TODO This transition is wrong. Should be a noop / error.
@@ -196,10 +196,10 @@ public class AMContainerImpl implements AMContainer {
     this.taskAttemptListener = tal;
     this.failedAssignments = new LinkedList<TezTaskAttemptID>();
 
-    this.noAllocationContainerTask = WAIT_TASK; 
+    this.noAllocationContainerTask = WAIT_TASK;
     this.stateMachine = stateMachineFactory.make(this);
   }
-  
+
   @Override
   public AMContainerState getState() {
     readLock.lock();
@@ -214,7 +214,7 @@ public class AMContainerImpl implements AMContainer {
   public ContainerId getContainerId() {
     return this.containerId;
   }
-  
+
   @Override
   public Container getContainer() {
     return this.container;
@@ -262,7 +262,7 @@ public class AMContainerImpl implements AMContainer {
       readLock.unlock();
     }
   }
-  
+
   @Override
   public int getShufflePort() {
     readLock.lock();
@@ -272,17 +272,19 @@ public class AMContainerImpl implements AMContainer {
       readLock.unlock();
     }
   }
-  
+
   public boolean isInErrorState() {
     return inError;
   }
 
   @Override
   public void handle(AMContainerEvent event) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Processing AMContainerEvent " + event.getContainerId()
+          + " of type " + event.getType() + " while in state: " + getState()
+          + ". Event: " + event);
+    }
     this.writeLock.lock();
-    LOG.info("DEBUG: Processing AMContainerEvent " + event.getContainerId()
-        + " of type " + event.getType() + " while in state: " + getState()
-        + ". Event: " + event);
     try {
       final AMContainerState oldState = getState();
       try {
@@ -302,7 +304,7 @@ public class AMContainerImpl implements AMContainer {
       writeLock.unlock();
     }
   }
-  
+
   @SuppressWarnings("unchecked")
   private void sendEvent(Event<?> event) {
     this.eventHandler.handle(event);
@@ -348,14 +350,14 @@ public class AMContainerImpl implements AMContainer {
           container.taskAttemptListener, event.getCredentials(),
           event.shouldProfile(), container.appContext);
 
-      // Registering now, so that in case of delayed NM response, the child 
+      // Registering now, so that in case of delayed NM response, the child
       // task is not told to die since the TAL does not know about the container.
       container.registerWithTAListener();
       container.sendStartRequestToNM();
       LOG.info("Sending Launch Request for Container with id: " +
           container.container.getId());
       // Forget about the clc to save resources. At some point, part of the clc
-      // info may need to be exposed to the scheduler to figure out whether a 
+      // info may need to be exposed to the scheduler to figure out whether a
       // container can be used for a specific TaskAttempt.
       container.clc = null;
     }
@@ -451,7 +453,9 @@ public class AMContainerImpl implements AMContainer {
         return AMContainerState.STOP_REQUESTED;
       }
       container.pendingAttempt = event.getTaskAttemptId();
-      LOG.info("DEBUG: AssignTA: attempt: " + event.getRemoteTaskContext());
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("AssignTA: attempt: " + event.getRemoteTaskContext());
+      }
       container.remoteTaskMap
           .put(event.getTaskAttemptId(), event.getRemoteTaskContext());
       return container.getState();
@@ -580,7 +584,7 @@ public class AMContainerImpl implements AMContainer {
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       super.transition(container, cEvent);
       if (container.pendingAttempt != null) {
-        container.sendTerminatingToTaskAttempt(container.pendingAttempt, 
+        container.sendTerminatingToTaskAttempt(container.pendingAttempt,
             "Container " + container.getContainerId() +
                 " hit an invalid transition - " + cEvent.getType() + " at " +
                 container.getState());
@@ -595,8 +599,10 @@ public class AMContainerImpl implements AMContainer {
     @Override
     public AMContainerState transition(
         AMContainerImpl container, AMContainerEvent cEvent) {
-      LOG.info("DEBUG: AssignTAAtIdle: attempt: " +
-          ((AMContainerEventAssignTA) cEvent).getRemoteTaskContext());
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("AssignTAAtIdle: attempt: " +
+            ((AMContainerEventAssignTA) cEvent).getRemoteTaskContext());
+      }
       return super.transition(container, cEvent);
     }
   }
@@ -617,12 +623,13 @@ public class AMContainerImpl implements AMContainer {
         if (container.lastTaskFinishTime != 0) {
           long idleTimeDiff =
               System.currentTimeMillis() - container.lastTaskFinishTime;
-          LOG.info("DEBUG: Computing idle time for container: " +
-              container.getContainerId() + ", lastFinishTime: " +
-              container.lastTaskFinishTime + ", Incremented by: " +
-              idleTimeDiff);
-          container.idleTimeBetweenTasks +=
-              System.currentTimeMillis() - container.lastTaskFinishTime;
+          container.idleTimeBetweenTasks += idleTimeDiff;
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Computing idle time for container: " +
+                container.getContainerId() + ", lastFinishTime: " +
+                container.lastTaskFinishTime + ", Incremented by: " +
+                idleTimeDiff);
+          }
         }
         LOG.info("Assigned taskAttempt + [" + container.runningAttempt +
             "] to container: [" + container.getContainerId() + "]");
@@ -656,7 +663,9 @@ public class AMContainerImpl implements AMContainer {
     @Override
     public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
       super.transition(container, cEvent);
-      LOG.info("DEBUG: IdleTimeBetweenTasks: " + container.idleTimeBetweenTasks);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("IdleTimeBetweenTasks: " + container.idleTimeBetweenTasks);
+      }
       container.unregisterFromContainerListener();
     }
   }
@@ -785,7 +794,7 @@ public class AMContainerImpl implements AMContainer {
       container.registerFailedAttempt(event.getTaskAttemptId());
     }
   }
-  
+
   // Hack to some extent. This allocation should be done while entering one of
   // the post-running states, insetad of being a transition on the post stop
   // states.
@@ -907,11 +916,11 @@ public class AMContainerImpl implements AMContainer {
   protected void registerFailedAttempt(TezTaskAttemptID taId) {
     failedAssignments.add(taId);
   }
-  
+
   protected void deAllocate() {
     sendEvent(new AMSchedulerEventDeallocateContainer(containerId));
   }
-  
+
   protected void sendCompletedToScheduler() {
     sendEvent(new AMSchedulerEventContainerCompleted(containerId));
   }
@@ -944,7 +953,7 @@ public class AMContainerImpl implements AMContainer {
     sendEvent(new NMCommunicatorStopRequestEvent(containerId,
         container.getNodeId(), container.getContainerToken()));
   }
-  
+
   protected void unregisterAttemptFromListener(TezTaskAttemptID attemptId) {
     taskAttemptListener.unregisterTaskAttempt(attemptId);
   }
@@ -965,7 +974,7 @@ public class AMContainerImpl implements AMContainer {
   protected void unregisterFromContainerListener() {
     this.containerHeartbeatHandler.unregister(this.containerId);
   }
-  
 
-  
+
+
 }


[03/50] [abbrv] git commit: TEZ-25: Change DAG representation from Configuration object to structured protobuf message

Posted by ss...@apache.org.
TEZ-25: Change DAG representation from Configuration object to structured protobuf message


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/cb5758b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/cb5758b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/cb5758b4

Branch: refs/heads/master
Commit: cb5758b424a80c5a17e0d9983a7b6ff2982dcd31
Parents: 97e1fa9
Author: mikelid <mi...@microsoft.com>
Authored: Mon May 13 11:21:27 2013 -0700
Committer: mikelid <mi...@microsoft.com>
Committed: Mon May 13 11:21:27 2013 -0700

----------------------------------------------------------------------
 pom.xml                                            |    5 +
 tez-dag-api/pom.xml                                |   26 +
 .../src/main/java/org/apache/tez/dag/api/DAG.java  |  128 ++++-
 .../org/apache/tez/dag/api/DAGConfiguration.java   |  492 ---------------
 .../org/apache/tez/dag/api/DAGLocationHint.java    |   95 ---
 .../org/apache/tez/dag/api/DagTypeConverters.java  |  221 +++++++
 .../org/apache/tez/dag/api/TezConfiguration.java   |    3 +-
 .../org/apache/tez/dag/api/VertexLocationHint.java |   83 +---
 tez-dag-api/src/main/proto/DAGPlan.proto           |  119 ++++
 .../java/org/apache/tez/dag/api/TestDAGPlan.java   |   82 +++
 .../java/org/apache/tez/dag/app/DAGAppMaster.java  |   82 ++--
 .../org/apache/tez/dag/app/MRRExampleHelper.java   |   33 +-
 .../main/java/org/apache/tez/dag/app/dag/DAG.java  |    7 +-
 .../java/org/apache/tez/dag/app/dag/Vertex.java    |    3 +
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |  101 ++--
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |   33 +-
 .../apache/tez/dag/utils/TezEngineChildJVM.java    |    6 +
 .../apache/tez/dag/api/TestDAGLocationHint.java    |  114 ----
 .../apache/tez/dag/api/TestVertexLocationHint.java |  147 -----
 .../java/org/apache/tez/mapreduce/YARNRunner.java  |   50 +-
 20 files changed, 736 insertions(+), 1094 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 586eb18..5c545c5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -248,6 +248,11 @@
         <artifactId>avro</artifactId>
         <version>1.5.3</version>
       </dependency>
+      <dependency>
+       <groupId>com.google.protobuf</groupId>
+       <artifactId>protobuf-java</artifactId>
+       <version>2.4.0a</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag-api/pom.xml b/tez-dag-api/pom.xml
index c084f01..2b6aebb 100644
--- a/tez-dag-api/pom.xml
+++ b/tez-dag-api/pom.xml
@@ -46,6 +46,32 @@
         <configuration>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-maven-plugins</artifactId>
+        <version>3.0.0-SNAPSHOT</version>
+        <executions>
+          <execution>
+            <id>compile-protoc</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>protoc</goal>
+            </goals>
+            <configuration>
+              <imports>
+                <param>${basedir}/src/main/proto</param>
+              </imports>
+              <source>
+                <directory>${basedir}/src/main/proto</directory>
+                <includes>
+                  <include>DAGPlan.proto</include>
+                </includes>
+              </source>
+              <output>${project.build.directory}/generated-sources/java</output>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 185f9a4..c18c720 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -18,8 +18,24 @@
 package org.apache.tez.dag.api;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.tez.dag.api.*;
+import org.apache.tez.dag.api.DAGPlan.PlanKeyValuePair.Builder;
+import org.apache.tez.dag.api.DAGPlan.PlanTaskLocationHint;
+import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
+import org.apache.tez.dag.api.EdgeProperty.SourceType;
+import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
+import org.apache.tez.dag.api.DAGPlan.*;
+
 
 public class DAG { // FIXME rename to Topology
   List<Vertex> vertices;
@@ -76,28 +92,102 @@ public class DAG { // FIXME rename to Topology
     //for each vertex if not specified?
   }
     
-  // FIXME DAGConfiguration is not public API
-  public DAGConfiguration serializeDag() {
-    DAGConfiguration dagConf = new DAGConfiguration();
-    
-    dagConf.setName(name);
-    dagConf.setVertices(vertices);
-    dagConf.setEdgeProperties(edges);
-    
-    for(Vertex vertex : vertices) {
-      if(vertex.getInputVertices() != null) {
-        dagConf.setInputVertices(vertex.getVertexName(), vertex.getInputVertices());
-        dagConf.setInputEdgeIds(vertex.getVertexName(), vertex.getInputEdgeIds());
+  // create protobuf message describing DAG
+  public JobPlan createDag(){
+    JobPlan.Builder jobBuilder = JobPlan.newBuilder();
+
+    jobBuilder.setName(this.name);
+
+    for(Vertex vertex : vertices){
+      VertexPlan.Builder vertexBuilder = VertexPlan.newBuilder();
+      vertexBuilder.setName(vertex.getVertexName());
+      vertexBuilder.setType(PlanVertexType.NORMAL); // vertex type is implicitly NORMAL until  TEZ-46.
+      vertexBuilder.setProcessorName(vertex.getProcessorName());
+
+      //task config
+      PlanTaskConfiguration.Builder taskConfigBuilder = PlanTaskConfiguration.newBuilder();
+      Resource resource = vertex.getTaskResource();
+      taskConfigBuilder.setNumTasks(vertex.getParallelism());
+      taskConfigBuilder.setMemoryMb(resource.getMemory());
+      taskConfigBuilder.setVirtualCores(resource.getVirtualCores());
+      taskConfigBuilder.setJavaOpts(vertex.getJavaOpts());
+
+      taskConfigBuilder.setTaskModule(vertex.getVertexName());
+      PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder();
+      Map<String,LocalResource> lrs = vertex.getTaskLocalResources();
+      for(String key : lrs.keySet()){
+        LocalResource lr = lrs.get(key);
+        localResourcesBuilder.setName(key);
+        localResourcesBuilder.setUri(DagTypeConverters.convertToDAGPlan(lr.getResource()));
+        localResourcesBuilder.setSize(lr.getSize());
+        localResourcesBuilder.setTimeStamp(lr.getTimestamp());
+        localResourcesBuilder.setType(DagTypeConverters.convertToDAGPlan(lr.getType()));
+        localResourcesBuilder.setVisibility(DagTypeConverters.convertToDAGPlan(lr.getVisibility()));
+        if(lr.getType() == LocalResourceType.PATTERN){
+          assert lr.getPattern() != null : "resourceType=PATTERN but pattern is null";
+          localResourcesBuilder.setPattern(lr.getPattern());
+        }
+        taskConfigBuilder.addLocalResource(localResourcesBuilder);
       }
-      if(vertex.getOutputVertices() != null) {
-        dagConf.setOutputVertices(vertex.getVertexName(), vertex.getOutputVertices());
-        dagConf.setOutputEdgeIds(vertex.getVertexName(), vertex.getOutputEdgeIds());
+
+      if(vertex.getTaskEnvironment() != null){
+        for(String key : vertex.getTaskEnvironment().keySet()){
+          PlanKeyValuePair.Builder envSettingBuilder = PlanKeyValuePair.newBuilder();
+          envSettingBuilder.setKey(key);
+          envSettingBuilder.setValue(vertex.getTaskEnvironment().get(key));
+          taskConfigBuilder.addEnvironmentSetting(envSettingBuilder);
+        }
       }
+
+      if(vertex.getTaskLocationsHint() != null ){
+        if(vertex.getTaskLocationsHint().getTaskLocationHints() != null){
+          for(TaskLocationHint hint : vertex.getTaskLocationsHint().getTaskLocationHints()){
+            PlanTaskLocationHint.Builder taskLocationHintBuilder = PlanTaskLocationHint.newBuilder();
+
+            if(hint.getDataLocalHosts() != null){
+              taskLocationHintBuilder.addAllHost(Arrays.asList(hint.getDataLocalHosts()));
+            }
+            if(hint.getRacks() != null){
+              taskLocationHintBuilder.addAllRack(Arrays.asList(hint.getRacks()));
+            }
+
+            vertexBuilder.addTaskLocationHint(taskLocationHintBuilder);
+          }
+        }
+      }
+
+      for(String inEdgeId : vertex.getInputEdgeIds()){
+        vertexBuilder.addInEdgeId(inEdgeId);
+      }
+      
+      for(String outEdgeId : vertex.getOutputEdgeIds()){
+        vertexBuilder.addOutEdgeId(outEdgeId);
+      }
+      
+      vertexBuilder.setTaskConfig(taskConfigBuilder);
+      jobBuilder.addVertex(vertexBuilder);
+    }
+
+    for(Edge edge : edges){
+      EdgePlan.Builder edgeBuilder = EdgePlan.newBuilder();
+      edgeBuilder.setId(edge.getId());
+      edgeBuilder.setInputVertexName(edge.getInputVertex().getVertexName());
+      edgeBuilder.setOutputVertexName(edge.getOutputVertex().getVertexName());
+      edgeBuilder.setConnectionPattern(DagTypeConverters.convertToDAGPlan(edge.getEdgeProperty().connectionPattern));
+      edgeBuilder.setSourceType(DagTypeConverters.convertToDAGPlan(edge.getEdgeProperty().getSourceType()));
+      edgeBuilder.setInputClass(edge.getEdgeProperty().inputClass);
+      edgeBuilder.setOutputClass(edge.getEdgeProperty().outputClass);
+
+      jobBuilder.addEdge(edgeBuilder);
     }
     
-    dagConf.setConfig(config);
-    
-    return dagConf;
+    for(Entry<String, String> entry : this.config.entrySet()){
+      PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
+      kvp.setKey(entry.getKey());
+      kvp.setValue(entry.getValue());
+      jobBuilder.addJobSetting(kvp);
+    }
+
+    return jobBuilder.build();
   }
-  
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
deleted file mode 100644
index 93ed118..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
+++ /dev/null
@@ -1,492 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api;
-
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.StringTokenizer;
-import java.util.TreeMap;
-import java.util.Map.Entry;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
-
-// FIXME dag conf should not be public API???
-public class DAGConfiguration extends Configuration {
-
-  private static final Log LOG = LogFactory.getLog(DAGConfiguration.class);
-
-  public DAGConfiguration(Configuration conf) {
-    super(conf);
-    if (! (conf instanceof DAGConfiguration)) {
-      this.reloadConfiguration();
-    }
-  }
-
-  public DAGConfiguration() {
-    super();
-  }
-
-  public final static String DAG = "tez.dag.";
-  
-  public final static String DAG_AM = DAG + "am.";
-
-  public final static String VERTEX = DAG + "vertex.";
-
-  public final static String TASK = DAG + "task.";
-
-  public final static String TASK_ATTEMPT = DAG + "attempt.";
-
-  public final static String TEZ_DAG_VERTICES = DAG + "vertices";
-
-  public final static String TEZ_DAG_EDGES = DAG + "edges";
-
-  public final static String EDGE = DAG + "edge.";
-
-  private final static String SEPARATOR = "|";
-  
-  public final static String TEZ_DAG_CONFIG_KEYS = DAG + "keys";
-  public final static String TEZ_DAG_CONFIG_VALUES = DAG + "values";
-
-  @Private
-  public void setConfig(Map<String, String> config) {
-    if(!config.isEmpty()) {
-      String[] key = new String[config.size()];
-      String[] value = new String[config.size()];
-      int i=0;
-      for(Entry<String, String> entry : config.entrySet()) {
-        key[i] = entry.getKey();
-        value[i] = entry.getValue();
-        i++;
-      }
-      setStrings(TEZ_DAG_CONFIG_KEYS, key);
-      setStrings(TEZ_DAG_CONFIG_VALUES, value);
-    }
-  }
-  
-  @Private
-  public Map<String, String> getConfig() {
-    HashMap<String, String> config = new HashMap<String, String>();
-    String[] key = getStrings(TEZ_DAG_CONFIG_KEYS);
-    if(key != null) {
-      String[] value = getStrings(TEZ_DAG_CONFIG_VALUES);
-      assert value.length == key.length;
-      for(int i=0; i<key.length; ++i) {
-        config.put(key[i], value[i]);
-      }
-    }
-    return config;
-  }
-  
-  @Private
-  public void setEdgeProperties(List<Edge> edges) {
-    String[] edgeIds = new String[edges.size()];
-    for(int i=0; i<edges.size(); ++i) {
-      edgeIds[i] = edges.get(i).getId();
-    }
-    setStrings(TEZ_DAG_EDGES, edgeIds);
-    for (Edge edge : edges) {
-      setEdgeProperty(edge.getEdgeProperty(), edge.getId());
-    }
-  }
-
-  public Map<String, EdgeProperty> getEdgeProperties() {
-    String[] edgeIds = getStrings(TEZ_DAG_EDGES);
-    if (edgeIds == null) {
-      return new TreeMap<String, EdgeProperty>();
-    }
-    Map<String, EdgeProperty> edgeProperties =
-                          new HashMap<String, EdgeProperty>(edgeIds.length);
-    for(int i=0; i<edgeIds.length; ++i) {
-      edgeProperties.put(edgeIds[i], getEdgeProperty(edgeIds[i]));
-    }
-    return edgeProperties;
-  }
-
-  private void setEdgeProperty(EdgeProperty edgeProperty, String edgeId) {
-    String[] edgeStrs = new String[4];
-    edgeStrs[0] = edgeProperty.getConnectionPattern().name();
-    edgeStrs[1] = edgeProperty.getSourceType().name();
-    edgeStrs[2] = edgeProperty.inputClass;
-    edgeStrs[3] = edgeProperty.outputClass;
-
-    setStrings(EDGE + edgeId, edgeStrs);
-  }
-
-  private EdgeProperty getEdgeProperty(String edgeId) {
-    String[] edgeStr = getStrings(EDGE + edgeId);
-    assert edgeStr.length == 4;
-    return new EdgeProperty(EdgeProperty.ConnectionPattern.valueOf(edgeStr[0]),
-                             EdgeProperty.SourceType.valueOf(edgeStr[1]),
-                             edgeStr[2],
-                             edgeStr[3]);
-  }
-
-  public final static String TEZ_DAG_VERTEX_TASKS = VERTEX + "num-tasks";
-  public final static int DEFAULT_TEZ_DAG_VERTEX_TASKS  = 0;
-
-  public int getNumVertexTasks(String vertexName) {
-    return getInt(
-        TEZ_DAG_VERTEX_TASKS + "." + vertexName,
-        DEFAULT_TEZ_DAG_VERTEX_TASKS);
-  }
-
-  public void setNumVertexTasks(String vertexName, int numTasks) {
-    setInt(
-        TEZ_DAG_VERTEX_TASKS + "." + vertexName,
-        numTasks);
-  }
-
-  public final String TEZ_DAG_VERTEX_TASK_MEMORY = TASK + "memory-mb";
-
-  public final int DEFAULT_TEZ_DAG_VERTEX_TASK_MEMORY = 1024;
-
-  public int getVertexTaskMemory(String vertexName) {
-    return getInt(
-        TEZ_DAG_VERTEX_TASK_MEMORY + "." + vertexName,
-        DEFAULT_TEZ_DAG_VERTEX_TASK_MEMORY);
-  }
-
-  public void setVertexTaskMemory(String vertexName, int memory) {
-    setInt(
-        TEZ_DAG_VERTEX_TASK_MEMORY + "." + vertexName,
-        memory);
-  }
-
-  public final String TEZ_DAG_VERTEX_TASK_CPU = TASK + "cpu-vcores";
-
-  public final int DEFAULT_TEZ_DAG_VERTEX_TASK_CORES = 1;
-
-  public int getVertexTaskCores(String vertexName) {
-    return getInt(
-        TEZ_DAG_VERTEX_TASK_CPU + "." + vertexName,
-        DEFAULT_TEZ_DAG_VERTEX_TASK_CORES);
-  }
-  public void setVertexTaskCores(String vertexName, int cores) {
-    setInt(
-        TEZ_DAG_VERTEX_TASK_CPU + "." + vertexName,
-        cores);
-  }
-
-  private final String[] EMPTY = new String[0];
-
-  public String[] getVertices() {
-    String[] vertices = getStrings(TEZ_DAG_VERTICES, EMPTY);
-    return vertices == null? EMPTY : vertices;
-  }
-
-  void setVertexResource(Vertex vertex) {
-    Resource resource = vertex.getTaskResource();
-    if (resource == null) {
-      return;
-    }
-    setVertexTaskCores(vertex.getVertexName(), resource.getVirtualCores());
-    setVertexTaskMemory(vertex.getVertexName(), resource.getMemory());
-  }
-
-  public Resource getVertexResource(String vertexName) {
-    int memory = getVertexTaskMemory(vertexName);
-    int vCores = getVertexTaskCores(vertexName);
-    return BuilderUtils.newResource(memory, vCores);
-  }
-
-  // FIXME we are serializing YarnURL which is not same as serializing a URL
-  public final String TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE = TASK + "local-resource.";
-  void setVertexLocalResource(Vertex vertex) {
-    Map<String,LocalResource> lrs = vertex.getTaskLocalResources();
-    if (lrs == null) {
-      return;
-    }
-    String[] lrStrs = new String[lrs.size()];
-    int i=0;
-    for(Map.Entry<String,LocalResource> entry : lrs.entrySet()) {
-      LocalResource lr = entry.getValue();
-      try {
-        String lrStr = StringUtils.escapeString(entry.getKey(),
-            StringUtils.ESCAPE_CHAR,
-            SEPARATOR.charAt(0))
-            + SEPARATOR
-            + StringUtils.escapeString(
-                ConverterUtils.getPathFromYarnURL(lr.getResource()).toString(),
-                StringUtils.ESCAPE_CHAR,
-                SEPARATOR.charAt(0))
-            + SEPARATOR
-            + StringUtils.escapeString(String.valueOf(lr.getSize()),
-                StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
-            + SEPARATOR
-            + StringUtils.escapeString(String.valueOf(lr.getTimestamp()),
-                StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
-            + SEPARATOR
-            + StringUtils.escapeString(lr.getType().name(),
-                StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
-            + SEPARATOR
-            + StringUtils.escapeString(lr.getVisibility().name(),
-                StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
-            + SEPARATOR
-            + StringUtils.escapeString(
-                (lr.getPattern() == null ? "" : lr.getPattern()),
-                StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
-        lrStrs[i++] = StringUtils.escapeString(lrStr);
-      } catch (URISyntaxException e) {
-        throw new RuntimeException(e);
-      }
-    }
-    setStrings(TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE + vertex.getVertexName(),
-        lrStrs);
-  }
-
-  public Map<String, LocalResource> getVertexLocalResources(String vertexName) {
-    String[] lrStrs = StringUtils.split(get(
-        TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE + vertexName, ""));
-    Map<String, LocalResource> localResources =
-        new TreeMap<String, LocalResource>();
-    if (lrStrs == null) {
-      return localResources;
-    }
-    LOG.info("XXXX Found " + lrStrs.length + " local resources");
-    for (String lrStr : lrStrs) {
-      String[] tokens =
-          StringUtils.split(
-              lrStr, StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
-      if (tokens.length != 6
-          && tokens.length != 7) {
-        LOG.warn("Invalid token count in serialized LocalResource"
-            + ", serializedString=" + lrStr
-            + ", tokenCount=" + tokens.length);
-      }
-      String resourceName = tokens[0];
-      LocalResource lRsrc = Records.newRecord(LocalResource.class);
-      lRsrc.setResource(ConverterUtils.getYarnUrlFromPath(
-          new Path(tokens[1])));
-      lRsrc.setSize(Long.valueOf(tokens[2]));
-      lRsrc.setTimestamp(Long.valueOf(tokens[3]));
-      lRsrc.setType(LocalResourceType.valueOf(tokens[4]));
-      lRsrc.setVisibility(LocalResourceVisibility.valueOf(
-          tokens[5]));
-      if (tokens.length == 7) {
-        lRsrc.setPattern(tokens[6]);
-      }
-      try {
-        LOG.info("XXXX Adding local resource"
-            + ", vertexName=" + vertexName
-            + ", resourceName=" + resourceName
-            + ", resourceUrl"
-            + ConverterUtils.getPathFromYarnURL(
-                lRsrc.getResource()).toString());
-      } catch (URISyntaxException e) {
-        // Ignore
-        // FIXME
-      }
-      localResources.put(resourceName, lRsrc);
-    }
-    return localResources;
-  }
-
-  public final String TEZ_DAG_VERTEX_TASK_ENV = TASK + "env.";
-  void setVertexEnv(Vertex vertex) {
-    Map<String, String> env = vertex.getTaskEnvironment();
-    if (env == null) {
-      return;
-    }
-    String[] envStrs = new String[env.size()];
-    int i=0;
-    for(Map.Entry<String,String> entry : env.entrySet()) {
-      String envStr = entry.getKey() + SEPARATOR + entry.getValue();
-      envStrs[i++] = StringUtils.escapeString(envStr);
-    }
-    set(TEZ_DAG_VERTEX_TASK_ENV + vertex.getVertexName(),
-        StringUtils.join(",", envStrs));
-  }
-
-  public Map<String,String> getVertexEnv(String vertexName) {
-    String[] envStrs = StringUtils.split(
-        get(TEZ_DAG_VERTEX_TASK_ENV + vertexName, ""));
-    Map<String,String> env = new HashMap<String,String>();
-    if(envStrs == null) {
-      return env;
-    }
-
-    LOG.info("XXXX Found " + envStrs.length + " environment");
-    for(String envStr : envStrs ) {
-      LOG.info("XXXX Parsing env from " + envStr);
-      StringTokenizer tokenizer = new StringTokenizer (envStr, SEPARATOR);
-      String envName = tokenizer.nextToken();
-      String envValue = tokenizer.nextToken();
-      env.put(StringUtils.unEscapeString(envName),
-          StringUtils.unEscapeString(envValue));
-    }
-
-    return env;
-  }
-  
-  public final String TEZ_DAG_NAME = DAG + "name";
-  @Private
-  public void setName(String name) {
-    set(TEZ_DAG_NAME, name);
-  }
-  
-  @Public
-  @Stable
-  public String getName() {
-    String name = get(TEZ_DAG_NAME);
-    return name;
-  }
-  
-  @Private
-  public void setVertices(List<Vertex> vertices) {
-    setVertices(TEZ_DAG_VERTICES, vertices);
-    for(Vertex vertex : vertices) {
-      // set num tasks
-      setNumVertexTasks(vertex.getVertexName(), vertex.getParallelism());
-      // set resource
-      setVertexResource(vertex);
-      // set localResource
-      setVertexLocalResource(vertex);
-      // set environment
-      setVertexEnv(vertex);
-      // set processor name
-      setVertexTaskModuleClassName(vertex);
-      //set javaOpts
-      setVertexJavaOpts(vertex.getVertexName(), vertex.getJavaOpts());
-    }
-  }
-
-  public final String TEZ_DAG_VERTEX_INPUT_VERTICES = VERTEX + "input-vertices";
-  @Public
-  @Stable
-  public String[] getInputVertices(String vertexName) {
-    String[] vertices =
-        getStrings(TEZ_DAG_VERTEX_INPUT_VERTICES + "." + vertexName, EMPTY);
-    return vertices == null ? EMPTY : vertices;
-  }
-  @Private
-  public void setInputVertices(String vertexName, List<Vertex> inputVertices) {
-    setVertices(TEZ_DAG_VERTEX_INPUT_VERTICES + "." + vertexName,
-        inputVertices);
-  }
-
-  private void setVertices(String key, List<Vertex> vertices) {
-    String[] verticesNames = new String[vertices.size()];
-    for (int i = 0; i < vertices.size(); ++i) {
-      verticesNames[i] = vertices.get(i).getVertexName();
-    }
-    setStrings(key, verticesNames);
-  }
-
-  public final String TEZ_DAG_VERTEX_OUTPUT_VERTICES = VERTEX
-      + "output-vertices";
-  @Public
-  @Stable
-  public String[] getOutputVertices(String vertexName) {
-    String[] vertices =
-        getStrings(TEZ_DAG_VERTEX_OUTPUT_VERTICES + "." + vertexName, EMPTY);
-    return vertices == null? EMPTY : vertices;
-  }
-  @Private
-  public void setOutputVertices(String vertexName,
-      List<Vertex> outputVertices) {
-    setVertices(TEZ_DAG_VERTEX_OUTPUT_VERTICES + "." + vertexName,
-        outputVertices);
-  }
-
-  public final String TEZ_DAG_VERTEX_INPUT_EDGES = VERTEX + "input-edges";
-  @Public
-  @Stable
-  public List<String> getInputEdgeIds(String vertexName) {
-    return getEdgeIds(TEZ_DAG_VERTEX_INPUT_EDGES + "." + vertexName);
-  }
-  public void setInputEdgeIds(String vertexName, List<String> edgeIds) {
-    setEdgeIds(TEZ_DAG_VERTEX_INPUT_EDGES + "." + vertexName, edgeIds);
-  }
-
-  public final String TEZ_DAG_VERTEX_OUTPUT_EDGES = VERTEX + "output-edges";
-  @Public
-  @Stable
-  public List<String> getOutputEdgeIds(String vertexName) {
-    return getEdgeIds(TEZ_DAG_VERTEX_OUTPUT_EDGES + "." + vertexName);
-  }
-  @Private
-  public void setOutputEdgeIds(String vertexName, List<String> edgeIds) {
-    setEdgeIds(TEZ_DAG_VERTEX_OUTPUT_EDGES + "." + vertexName, edgeIds);
-  }
-
-  private List<String> getEdgeIds(String key) {
-    String[] edgeIds = getStrings(key, EMPTY);
-    if (edgeIds == null) {
-      return new ArrayList<String>();
-    }
-    return Arrays.asList(edgeIds);
-  }
-
-  private void setEdgeIds(String key, List<String> edgeIds) {
-    setStrings(key, edgeIds.toArray(new String[]{}));
-  }
-
-  private void setVertexTaskModuleClassName(Vertex vertex) {
-    setVertexTaskModuleClassName(vertex.getVertexName(),
-        vertex.getProcessorName());
-  }
-
-  public final String TEZ_DAG_VERTEX_TASK_MODULE= VERTEX + "task-module";
-  @Private
-  public String getVertexTaskModuleClassName(String vertexName) {
-    return get(TEZ_DAG_VERTEX_TASK_MODULE + "." + vertexName);
-  }
-  @Private
-  public void setVertexTaskModuleClassName(String vertexName,
-      String taskModule) {
-    set(TEZ_DAG_VERTEX_TASK_MODULE + "." + vertexName, taskModule);
-  }
-  
-  public final String TEZ_DAG_VERTEX_JAVAOPTS= VERTEX + "java-opts";
-  @Private 
-  public String getVertexJavaOpts(String vertexName) {
-	  String opts = get(TEZ_DAG_VERTEX_JAVAOPTS + "." + vertexName);
-	  return opts == null? "" : opts;
-  }
-  
-  @Private
-  public void setVertexJavaOpts(String vertexName, String javaOpts){
-	  set(TEZ_DAG_VERTEX_JAVAOPTS + "." + vertexName, javaOpts);
-  }
-
-  /// File used for storing location hints that are passed to the DAG
-  public static final String DAG_LOCATION_HINT_RESOURCE_FILE =
-      DAG + "location-hint-resource-file";
-  public static final String DEFAULT_DAG_LOCATION_HINT_RESOURCE_FILE =
-      "tezdaglocationhint.info";
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java
deleted file mode 100644
index 9f48f2c..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api;
-
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeMap;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-
-public class DAGLocationHint implements Writable {
-
-  private Map<String, VertexLocationHint> vertexLocationHints;
-
-  public DAGLocationHint() {
-    vertexLocationHints = new TreeMap<String, VertexLocationHint>();
-  }
-
-  public Map<String, VertexLocationHint> getVertexLocationHints() {
-    return vertexLocationHints;
-  }
-
-  public VertexLocationHint getVertexLocationHint(String vertexName) {
-    return vertexLocationHints.get(vertexName);
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(vertexLocationHints.size());
-    for (Entry<String, VertexLocationHint> entry :
-        vertexLocationHints.entrySet()) {
-      Text.writeString(out, entry.getKey());
-      out.writeBoolean(entry.getValue() != null);
-      if (entry.getValue() != null) {
-        entry.getValue().write(out);
-      }
-    }
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    int entryCount = in.readInt();
-    vertexLocationHints = new TreeMap<String, VertexLocationHint>();
-    for (int i = 0; i < entryCount; ++i) {
-      String vertexName = Text.readString(in);
-      if (!in.readBoolean()) {
-        vertexLocationHints.put(vertexName, null);
-      } else {
-        VertexLocationHint hint = new VertexLocationHint();
-        hint.readFields(in);
-        vertexLocationHints.put(vertexName, hint);
-      }
-    }
-  }
-
-  public static DAGLocationHint initDAGDagLocationHint(
-      String locationHintFile) throws IOException {
-    DataInput in = new DataInputStream(new FileInputStream(locationHintFile));
-    DAGLocationHint dagLocationHint = new DAGLocationHint();
-    dagLocationHint.readFields(in);
-    return dagLocationHint;
-  }
-
-  public static void writeDAGDagLocationHint(
-      DAGLocationHint dagLocationHint,
-      String locationHintFile) throws IOException {
-    DataOutput out = new DataOutputStream(new FileOutputStream(
-        locationHintFile));
-    dagLocationHint.write(out);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
new file mode 100644
index 0000000..04404ba
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.dag.api;
+
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.dag.api.DAGPlan.EdgePlan;
+import org.apache.tez.dag.api.DAGPlan.PlanKeyValuePair;
+import org.apache.tez.dag.api.DAGPlan.PlanLocalResource;
+import org.apache.tez.dag.api.DAGPlan.PlanTaskConfiguration;
+import org.apache.tez.dag.api.DAGPlan.PlanTaskLocationHint;
+import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
+import org.apache.tez.dag.api.EdgeProperty.SourceType;
+import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
+import org.apache.tez.dag.api.DAGPlan.*;
+
+public class DagTypeConverters {
+  
+  public static PlanLocalResourceVisibility convertToDAGPlan(LocalResourceVisibility visibility){
+    switch(visibility){
+      case PUBLIC : return PlanLocalResourceVisibility.PUBLIC;  
+      case PRIVATE : return PlanLocalResourceVisibility.PRIVATE;
+      case APPLICATION : return PlanLocalResourceVisibility.APPLICATION;
+      default : throw new RuntimeException("unknown 'visibility'");
+    }
+  }
+  
+  public static LocalResourceVisibility convertFromDAGPlan(PlanLocalResourceVisibility visibility){
+    switch(visibility){
+      case PUBLIC : return LocalResourceVisibility.PUBLIC;  
+      case PRIVATE : return LocalResourceVisibility.PRIVATE;
+      case APPLICATION : return LocalResourceVisibility.APPLICATION;
+      default : throw new RuntimeException("unknown 'visibility'");
+    }
+  }
+  
+  public static PlanEdgeSourceType convertToDAGPlan(SourceType sourceType){
+    switch(sourceType){
+      case STABLE : return PlanEdgeSourceType.STABLE;  
+      case STABLE_PERSISTED : return PlanEdgeSourceType.STABLE_PERSISTED;
+      case STREAMING :  return PlanEdgeSourceType.STREAMING;
+      default : throw new RuntimeException("unknown 'sourceType'");
+    }
+  }
+  
+  public static SourceType convertFromDAGPlan(PlanEdgeSourceType sourceType){
+    switch(sourceType){
+      case STABLE : return SourceType.STABLE;  
+      case STABLE_PERSISTED : return SourceType.STABLE_PERSISTED;
+      case STREAMING :  return SourceType.STREAMING;
+      default : throw new RuntimeException("unknown 'sourceType'");
+    }
+  }
+  
+  public static PlanEdgeConnectionPattern convertToDAGPlan(ConnectionPattern pattern){
+    switch(pattern){
+      case ONE_TO_ONE : return PlanEdgeConnectionPattern.ONE_TO_ONE;  
+      case ONE_TO_ALL : return PlanEdgeConnectionPattern.ONE_TO_ALL;
+      case BIPARTITE : return PlanEdgeConnectionPattern.BIPARTITE;
+      default : throw new RuntimeException("unknown 'pattern'");
+    }
+  }
+  
+  public static ConnectionPattern convertFromDAGPlan(PlanEdgeConnectionPattern pattern){
+    switch(pattern){
+      case ONE_TO_ONE : return ConnectionPattern.ONE_TO_ONE;  
+      case ONE_TO_ALL : return ConnectionPattern.ONE_TO_ALL;
+      case BIPARTITE : return ConnectionPattern.BIPARTITE;
+      default : throw new IllegalArgumentException("unknown 'pattern'");
+    }
+  }
+  
+  public static PlanLocalResourceType convertToDAGPlan(LocalResourceType type) {
+    switch(type){
+    case ARCHIVE : return PlanLocalResourceType.ARCHIVE;
+    case FILE : return PlanLocalResourceType.FILE;
+    case PATTERN : return PlanLocalResourceType.PATTERN;
+    default : throw new IllegalArgumentException("unknown 'type'");
+    }
+  }
+  
+  public static LocalResourceType convertFromDAGPlan(PlanLocalResourceType type) {
+    switch(type){
+    case ARCHIVE : return LocalResourceType.ARCHIVE;
+    case FILE : return LocalResourceType.FILE;
+    case PATTERN : return LocalResourceType.PATTERN;
+    default : throw new IllegalArgumentException("unknown 'type'");
+    }
+  }
+
+  public static VertexLocationHint convertFromDAGPlan(
+      List<PlanTaskLocationHint> locationHints) {
+
+    List<TaskLocationHint> outputList = new ArrayList<TaskLocationHint>();  
+    
+    for(PlanTaskLocationHint inputHint : locationHints){
+      TaskLocationHint outputHint = new TaskLocationHint(); 
+      outputHint.setRacks(inputHint.getRackList().toArray(new String[inputHint.getRackList().size()]));
+      outputHint.setDataLocalHosts(inputHint.getHostList().toArray(new String[inputHint.getHostList().size()]));
+      outputList.add(outputHint);
+    }
+    return new VertexLocationHint(outputList.size(), outputList.toArray(new TaskLocationHint[outputList.size()]));
+  }
+
+  // notes re HDFS URL handling:
+  //   Resource URLs in the protobuf message are strings of the form hdfs://host:port/path 
+  //   org.apache.hadoop.fs.Path.Path  is actually a URI type that allows any scheme
+  //   org.apache.hadoop.yarn.api.records.URL is a URL type used by YARN.
+  //   java.net.URL cannot be used out of the box as it rejects unknown schemes such as HDFS.
+  
+  public static String convertToDAGPlan(URL resource) {
+    // see above notes on HDFS URL handling
+    String out = resource.getScheme() + "://" + resource.getHost() + ":" + resource.getPort() 
+        + resource.getFile();
+    return out;
+  }
+
+  public static Map<String, LocalResource> createLocalResourceMapFromDAGPlan(
+      List<PlanLocalResource> localResourcesList) {
+    Map<String, LocalResource> map = new HashMap<String, LocalResource>();
+    for(PlanLocalResource res : localResourcesList){
+      LocalResource r = new LocalResourcePBImpl();
+      
+      //NOTE: have to check every optional field in protobuf generated classes for existence before accessing
+      //else we will receive a default value back, eg ""
+      if(res.hasPattern()){
+        r.setPattern(res.getPattern());
+      }
+      r.setResource(ConverterUtils.getYarnUrlFromPath(new Path(res.getUri())));  // see above notes on HDFS URL handling
+      r.setSize(res.getSize());
+      r.setTimestamp(res.getTimeStamp());
+      r.setType(DagTypeConverters.convertFromDAGPlan(res.getType()));
+      r.setVisibility(DagTypeConverters.convertFromDAGPlan(res.getVisibility()));
+      map.put(res.getName(), r);
+    }
+    return map;
+  }
+
+  public static Map<String, String> createEnvironmentMapFromDAGPlan(
+      List<PlanKeyValuePair> environmentSettingList) {  
+      
+    Map<String, String> map = new HashMap<String, String>();
+    for(PlanKeyValuePair setting : environmentSettingList){
+      map.put(setting.getKey(), setting.getValue());
+    }
+    
+    return map;
+  }
+  
+  public static Map<String, EdgePlan> createEdgePlanMapFromDAGPlan(List<EdgePlan> edgeList){
+    Map<String, EdgePlan> edgePlanMap =
+        new HashMap<String, EdgePlan>();
+    for(EdgePlan edgePlanItem : edgeList){
+      edgePlanMap.put(edgePlanItem.getId(), edgePlanItem);
+    }
+    return edgePlanMap;
+  }
+  
+  public static Map<String, EdgeProperty> createEdgePropertyMapFromDAGPlan(
+      List<EdgePlan> edgeList) {  
+      
+    Map<String, EdgeProperty> map = new HashMap<String, EdgeProperty>();
+    for(EdgePlan edge: edgeList){
+       map.put(edge.getId(), 
+           new EdgeProperty(
+               convertFromDAGPlan(edge.getConnectionPattern()),
+               convertFromDAGPlan(edge.getSourceType()),
+               edge.getInputClass(),
+               edge.getOutputClass()
+               )
+           );
+    }
+    
+    return map;
+  }
+
+  public static Resource CreateResourceRequestFromTaskConfig(
+      PlanTaskConfiguration taskConfig) {
+    return BuilderUtils.newResource(taskConfig.getMemoryMb(), taskConfig.getVirtualCores());
+  }
+
+  public static Map<String, String> createSettingsMapFromDAGPlan(
+      List<PlanKeyValuePair> settingList) {
+    Map<String, String> map = new HashMap<String, String>();
+    for(PlanKeyValuePair setting: settingList){
+      map.put(setting.getKey(), setting.getValue());
+    }
+    return map;
+  }   
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index be03cd8..597fb2f 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -98,5 +98,6 @@ private static final String TEZ_CONF_DIR_ENV = "TEZ_CONF_DIR";
     TEZ_HOME_ENV + "/lib/*"
   };
 
-  public static final String DAG_AM_PLAN_CONFIG_XML = "tez-dag.xml";
+  public static final String DAG_AM_PLAN_PB_BINARY = "tez-dag.pb";
+  public static final String DAG_AM_PLAN_PB_TEXT = "tez-dag.pb.txt";
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
index 8571252..381b4ba 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
@@ -18,14 +18,7 @@
 
 package org.apache.tez.dag.api;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-
-public class VertexLocationHint implements Writable {
+public class VertexLocationHint  {
 
   private int numTasks;
   private TaskLocationHint[] taskLocationHints;
@@ -56,7 +49,7 @@ public class VertexLocationHint implements Writable {
     this.taskLocationHints = taskLocationHints;
   }
 
-  public static class TaskLocationHint implements Writable {
+  public static class TaskLocationHint {
 
     // Host names if any to be used
     private String[] hosts;
@@ -84,77 +77,5 @@ public class VertexLocationHint implements Writable {
     public void setRacks(String[] racks) {
       this.racks = racks;
     }
-
-    private void writeStringArray(DataOutput out, String[] array)
-        throws IOException {
-      if (array == null) {
-        out.writeInt(-1);
-        return;
-      }
-      out.writeInt(array.length);
-      for (String entry : array) {
-        out.writeBoolean(entry != null);
-        if (entry != null) {
-          Text.writeString(out, entry);
-        }
-      }
-    }
-
-    private String[] readStringArray(DataInput in)
-        throws IOException {
-      int arrayLen = in.readInt();
-      if (arrayLen == -1) {
-        return null;
-      }
-      String[] array = new String[arrayLen];
-      for (int i = 0; i < arrayLen; ++i) {
-        if (!in.readBoolean()) {
-          array[i] = null;
-        } else {
-          array[i] = Text.readString(in);
-        }
-      }
-      return array;
-    }
-
-    @Override
-    public void write(DataOutput out) throws IOException {
-      writeStringArray(out, hosts);
-      writeStringArray(out, racks);
-    }
-
-    @Override
-    public void readFields(DataInput in) throws IOException {
-      hosts = readStringArray(in);
-      racks = readStringArray(in);
-    }
-
-
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(numTasks);
-    for (int i = 0; i < numTasks; ++i) {
-      out.writeBoolean(taskLocationHints[i] != null);
-      if (taskLocationHints[i] != null) {
-        taskLocationHints[i].write(out);
-      }
-    }
   }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    numTasks = in.readInt();
-    taskLocationHints = new TaskLocationHint[numTasks];
-    for (int i = 0; i < numTasks; ++i) {
-      if (!in.readBoolean()) {
-        taskLocationHints[i] = null;
-      } else {
-        taskLocationHints[i] = new TaskLocationHint(null, null);
-        taskLocationHints[i].readFields(in);
-      }
-    }
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/proto/DAGPlan.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGPlan.proto b/tez-dag-api/src/main/proto/DAGPlan.proto
new file mode 100644
index 0000000..38e8971
--- /dev/null
+++ b/tez-dag-api/src/main/proto/DAGPlan.proto
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tez.dag.api";
+option java_outer_classname = "DAGPlan";
+option java_generate_equals_and_hash = true;
+package org.apache.tez.dag.api;
+
+// Many of these types have a dual in the Tez-api.  To reduce confusion, these types have prefix or suffix 
+// of "Plan" to indicate they are to be used in the dag-plan.
+// The big types use a suffix:  JobPlan, VertexPlan, EdgePlan 
+//   --> these get more direct use in the runtime and the naming is natural.
+// The enums and utility types use prefix: PlanVertexType, PlanEdgeConnectionPaatern, etc
+//   --> there is not great naming choice for these that avoids ambiguity, but this one seems acceptable. 
+
+enum PlanVertexType {
+  INPUT = 0;
+  NORMAL = 1;
+  OUTPUT = 2;
+}
+
+enum PlanEdgeConnectionPattern {
+  ONE_TO_ONE = 0;
+  ONE_TO_ALL = 1;
+  BIPARTITE = 2;
+}
+
+enum PlanEdgeSourceType {
+  STABLE = 0;
+  STABLE_PERSISTED = 1;
+  STREAMING = 2;
+}
+
+message PlanKeyValuePair {
+  required string key = 1;
+  required string value = 2;
+}
+
+enum PlanLocalResourceType {
+  FILE = 0;
+  ARCHIVE = 1;
+  PATTERN = 2;
+}
+
+enum PlanLocalResourceVisibility {
+  PUBLIC = 0;
+  PRIVATE = 1;
+  APPLICATION = 2;
+}
+
+message PlanLocalResource {
+  required string name = 1;
+  required string uri = 2;
+  required int64 size = 3;
+  required int64 timeStamp = 4;
+  required PlanLocalResourceType type = 5;
+  required PlanLocalResourceVisibility visibility = 6;
+  optional string pattern = 7; // only used if type=PATTERN
+}
+
+// Each taskLocationHint represents a single split in in the input.
+// It is the list of [{rack,machines}] that host a replica of each particular split.
+// For now it is represented as pair-of-arrays rather than array-of-pairs.
+message PlanTaskLocationHint {
+  repeated string rack = 1;
+  repeated string host = 2;
+}
+
+message PlanTaskConfiguration {
+  required int32 numTasks = 1;
+  required int32 memoryMb = 2;
+  required int32 virtualCores = 3;
+  required string javaOpts = 4;
+  required string taskModule = 5;
+  repeated PlanLocalResource localResource = 6;
+  repeated PlanKeyValuePair environmentSetting = 8;  
+}
+
+message VertexPlan {
+  required string name = 1;
+  required PlanVertexType type = 2;
+  optional string processorName = 3;
+  required PlanTaskConfiguration taskConfig = 4;
+  repeated PlanTaskLocationHint taskLocationHint = 7;
+  repeated string inEdgeId = 8;
+  repeated string outEdgeId = 9;
+}
+
+message EdgePlan {
+  required string id = 1;
+  required string inputVertexName = 2;
+  required string outputVertexName = 3;
+  required PlanEdgeConnectionPattern connectionPattern = 4;
+  required PlanEdgeSourceType sourceType = 5;
+  required string inputClass = 6;
+  required string outputClass = 7;
+}
+
+message JobPlan {
+  required string name = 1;
+  repeated VertexPlan vertex = 3;
+  repeated EdgePlan edge = 4;
+  repeated PlanKeyValuePair jobSetting = 5;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
new file mode 100644
index 0000000..f89a73e
--- /dev/null
+++ b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
+import org.apache.tez.dag.api.DAGPlan.*;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+// based on TestDAGLocationHint
+public class TestDAGPlan {
+  @Rule
+  public TemporaryFolder tempFolder = new TemporaryFolder(); //TODO: doesn't seem to be deleting this folder automatically as expected.
+
+  @Test
+  public void testBasicJobPlanSerde() throws IOException {
+
+    JobPlan job = JobPlan.newBuilder()
+       .setName("test")
+       .addVertex(
+           VertexPlan.newBuilder()
+             .setName("vertex1")
+             .setType(PlanVertexType.NORMAL)
+             .addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("machineName").addRack("rack1").build())
+             .setTaskConfig(
+                 DAGPlan.PlanTaskConfiguration.newBuilder()
+                   .setNumTasks(2)
+                   .setVirtualCores(4)
+                   .setMemoryMb(1024)
+                   .setJavaOpts("")
+                   .setTaskModule("x.y")
+                   .build())
+             .build())
+        .build();
+   File file = tempFolder.newFile("jobPlan");
+   FileOutputStream outStream = null;
+   try {
+     outStream = new FileOutputStream(file);
+     job.writeTo(outStream); 
+   }
+   finally {
+     if(outStream != null){
+       outStream.close();  
+     }
+   }
+
+   JobPlan inJob;
+   FileInputStream inputStream;
+   try {
+     inputStream = new FileInputStream(file);
+     inJob = JobPlan.newBuilder().mergeFrom(inputStream).build();
+   }
+   finally {
+     outStream.close();  
+   }
+
+   Assert.assertEquals(job, inJob);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 37587e4..57003c1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -19,6 +19,7 @@
 package org.apache.tez.dag.app;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.util.EnumSet;
@@ -62,8 +63,9 @@ import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.service.CompositeService;
 import org.apache.hadoop.yarn.service.Service;
 import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.DAGConfiguration;
-import org.apache.tez.dag.api.DAGLocationHint;
+import org.apache.tez.dag.api.DAGPlan.JobPlan;
+import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.app.client.ClientService;
 import org.apache.tez.dag.app.client.impl.TezClientService;
@@ -131,7 +133,7 @@ public class DAGAppMaster extends CompositeService {
   public static final int SHUTDOWN_HOOK_PRIORITY = 30;
 
   private Clock clock;
-  private final DAGConfiguration dagPlan;
+  private final JobPlan jobPlan;
   private long dagsStartTime;
   private final long startTime;
   private final long appSubmitTime;
@@ -170,8 +172,6 @@ public class DAGAppMaster extends CompositeService {
   private TaskSchedulerEventHandler taskSchedulerEventHandler;
   private HistoryEventHandler historyEventHandler;
 
-  private DAGLocationHint dagLocationHint;
-
   private DAGAppMasterState state;
 
   private DAG dag;
@@ -180,16 +180,16 @@ public class DAGAppMaster extends CompositeService {
 
   public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
       ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
-      long appSubmitTime, DAGConfiguration dagPlan) {
+      long appSubmitTime, JobPlan dagPB) {
     this(applicationAttemptId, containerId, nmHost, nmPort, nmHttpPort,
-        new SystemClock(), appSubmitTime, dagPlan);
+        new SystemClock(), appSubmitTime, dagPB);
   }
 
   public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
       ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
-      Clock clock, long appSubmitTime, DAGConfiguration dagPlan) {
+      Clock clock, long appSubmitTime, JobPlan dagPB) {
     super(DAGAppMaster.class.getName());
-    this.dagPlan = dagPlan;
+    this.jobPlan = dagPB;
     this.clock = clock;
     this.startTime = clock.getTime();
     this.appSubmitTime = appSubmitTime;
@@ -215,13 +215,12 @@ public class DAGAppMaster extends CompositeService {
     conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
 
     downloadTokensAndSetupUGI(conf);
-    setupDAGLocationHint(dagPlan);
 
     context = new RunningAppContext(conf);
 
     // Job name is the same as the app name util we support DAG of jobs
     // for an app later
-    appName = dagPlan.getName();
+    appName = jobPlan.getName();
 
     dagId = new TezDAGID(appAttemptID.getApplicationId(), 1);
 
@@ -521,11 +520,11 @@ public class DAGAppMaster extends CompositeService {
 //  }
 
   /** Create and initialize (but don't start) a single dag. */
-  protected DAG createDAG(DAGConfiguration dagPlan) {
+  protected DAG createDAG(JobPlan dagPB) {
 
     // create single job
     DAG newDag =
-        new DAGImpl(dagId, appAttemptID, conf, dagPlan, dispatcher.getEventHandler(),
+        new DAGImpl(dagId, appAttemptID, conf, dagPB, dispatcher.getEventHandler(),
             taskAttemptListener, jobTokenSecretManager, fsTokens, clock,
             // TODO Recovery
             //completedTasksFromPreviousRun,
@@ -534,7 +533,7 @@ public class DAGAppMaster extends CompositeService {
             //committer, newApiCommitter,
             currentUser.getShortUserName(), appSubmitTime,
             //amInfos,
-            taskHeartbeatHandler, context, dagLocationHint);
+            taskHeartbeatHandler, context);
     ((RunningAppContext) context).setDAG(newDag);
 
     dispatcher.register(DAGFinishEvent.Type.class,
@@ -578,23 +577,6 @@ public class DAGAppMaster extends CompositeService {
     }
   }
 
-  protected void setupDAGLocationHint(DAGConfiguration conf) {
-    try {
-      String dagLocationHintFile =
-          conf.get(DAGConfiguration.DAG_LOCATION_HINT_RESOURCE_FILE,
-              DAGConfiguration.DEFAULT_DAG_LOCATION_HINT_RESOURCE_FILE);
-      File f = new File(dagLocationHintFile);
-      if (f.exists()) {
-        this.dagLocationHint = DAGLocationHint.initDAGDagLocationHint(
-            dagLocationHintFile);
-      } else {
-        this.dagLocationHint = new DAGLocationHint();
-      }
-    } catch (IOException e) {
-      throw new YarnException(e);
-    }
-  }
-
   protected void addIfService(Object object) {
     if (object instanceof Service) {
       addService((Service) object);
@@ -846,7 +828,7 @@ public class DAGAppMaster extends CompositeService {
     */
     
     // /////////////////// Create the job itself.
-    dag = createDAG(dagPlan);
+    dag = createDAG(jobPlan);
 
     // End of creating the job.
 
@@ -973,27 +955,41 @@ public class DAGAppMaster extends CompositeService {
       // Default to running mr if nothing specified.
       // TODO change this once the client is ready.
       String type;
-      DAGConfiguration dagPlan = null;
       TezConfiguration conf = new TezConfiguration(new YarnConfiguration());
+      
+      JobPlan jobPlan = null;
       if (cliParser.hasOption(OPT_PREDEFINED)) {
         LOG.info("Running with PreDefined configuration");
         type = cliParser.getOptionValue(OPT_PREDEFINED, "mr");
         LOG.info("Running job type: " + type);
 
         if (type.equals("mr")) {
-          dagPlan = (DAGConfiguration)MRRExampleHelper.createDAGConfigurationForMR();
+          jobPlan = MRRExampleHelper.createDAGConfigurationForMR();
         } else if (type.equals("mrr")) {
-          dagPlan = (DAGConfiguration)MRRExampleHelper.createDAGConfigurationForMRR();
+          jobPlan = MRRExampleHelper.createDAGConfigurationForMRR();
+        }
+      } 
+      else {
+        // Read the protobuf DAG
+        JobPlan.Builder dagPlanBuilder = JobPlan.newBuilder(); 
+        FileInputStream dagPBBinaryStream = null;
+        try {
+          dagPBBinaryStream = new FileInputStream(TezConfiguration.DAG_AM_PLAN_PB_BINARY);
+          dagPlanBuilder.mergeFrom(dagPBBinaryStream);
         }
-      } else {
-        dagPlan = new DAGConfiguration();
-        dagPlan.addResource(TezConfiguration.DAG_AM_PLAN_CONFIG_XML);
+        finally {
+          if(dagPBBinaryStream != null){
+            dagPBBinaryStream.close();  
+          }
+        }
+
+        jobPlan = dagPlanBuilder.build();
       }
 
       LOG.info("XXXX Running a DAG with "
-          + dagPlan.getVertices().length + " vertices ");
-      for (String v : dagPlan.getVertices()) {
-        LOG.info("XXXX DAG has vertex " + v);
+          + jobPlan.getVertexCount() + " vertices ");
+      for (VertexPlan v : jobPlan.getVertexList()) {
+        LOG.info("XXXX DAG has vertex " + v.getName());
       }
 
       String jobUserName = System
@@ -1004,7 +1000,7 @@ public class DAGAppMaster extends CompositeService {
       // the objects myself.
       conf.setBoolean("fs.automatic.close", false);
       
-      Map<String, String> config = dagPlan.getConfig();
+      Map<String, String> config = DagTypeConverters.createSettingsMapFromDAGPlan(jobPlan.getJobSettingList());
       for(Entry<String, String> entry : config.entrySet()) {
         conf.set(entry.getKey(), entry.getValue());
       }
@@ -1012,7 +1008,7 @@ public class DAGAppMaster extends CompositeService {
       DAGAppMaster appMaster =
           new DAGAppMaster(applicationAttemptId, containerId, nodeHostString,
               Integer.parseInt(nodePortString),
-              Integer.parseInt(nodeHttpPortString), appSubmitTime, dagPlan);
+              Integer.parseInt(nodeHttpPortString), appSubmitTime, jobPlan);
       ShutdownHookManager.get().addShutdownHook(
         new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
index 1b69919..b50887a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
@@ -17,7 +17,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.dag.api.DAGConfiguration;
+import org.apache.tez.dag.api.DAGPlan.JobPlan;
 import org.apache.tez.dag.api.Edge;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
@@ -84,7 +84,10 @@ public class MRRExampleHelper {
    return resourceNames;
  }
 
- static Configuration createDAGConfigurationForMRR() throws IOException {
+ // TODO: these preconfigured jobs seem to require User and perhaps some other work.
+ //       -> not tested with new DagPB system.
+ 
+ static JobPlan createDAGConfigurationForMRR() throws IOException {
    org.apache.tez.dag.api.DAG dag = new org.apache.tez.dag.api.DAG();
    Vertex mapVertex = new Vertex("map",
        "org.apache.tez.mapreduce.task.InitialTask", 6);
@@ -143,16 +146,15 @@ public class MRRExampleHelper {
    dag.addEdge(edge1);
    dag.addEdge(edge2);
    dag.verify();
-   DAGConfiguration dagConf = dag.serializeDag();
-
-   dagConf.setBoolean(MRJobConfig.MAP_SPECULATIVE, false);
-   dagConf.setBoolean(MRJobConfig.REDUCE_SPECULATIVE, false);
-
-   return dagConf;
+   dag.addConfiguration(MRJobConfig.MAP_SPECULATIVE, new Boolean(false).toString());
+   dag.addConfiguration(MRJobConfig.REDUCE_SPECULATIVE, new Boolean(false).toString());
+   
+   JobPlan dagPB = dag.createDag();
+   return dagPB;
  }
 
  // TODO remove once client is in place
- static Configuration createDAGConfigurationForMR() throws IOException {
+ static JobPlan createDAGConfigurationForMR() throws IOException {
    org.apache.tez.dag.api.DAG dag = new org.apache.tez.dag.api.DAG();
    Vertex mapVertex = new Vertex("map",
        "org.apache.tez.mapreduce.task.InitialTask", 6);
@@ -194,12 +196,13 @@ public class MRRExampleHelper {
    dag.addVertex(reduceVertex);
    dag.addEdge(edge);
    dag.verify();
-   DAGConfiguration dagConf = dag.serializeDag();
-
-   dagConf.setBoolean(MRJobConfig.MAP_SPECULATIVE, false);
-   dagConf.setBoolean(MRJobConfig.REDUCE_SPECULATIVE, false);
-
-   return dagConf;
+   
+   dag.addConfiguration(MRJobConfig.MAP_SPECULATIVE, new Boolean(false).toString());
+   dag.addConfiguration(MRJobConfig.REDUCE_SPECULATIVE, new Boolean(false).toString());
+   
+   JobPlan dagPB = dag.createDag();
+   
+   return dagPB;
  }
   
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index 03bf570..39cfc1a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -24,7 +24,7 @@ import java.util.Map;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGConfiguration;
+import org.apache.tez.dag.api.DAGPlan.JobPlan;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.engine.records.TezDAGID;
@@ -60,8 +60,8 @@ public interface DAG {
   
   TezConfiguration getConf();
   
-  DAGConfiguration getDagPlan();
-
+  JobPlan getJobPlan();
+  
   /**
    * @return the ACLs for this job for each type of JobACL given. 
    */
@@ -75,5 +75,4 @@ public interface DAG {
   
   boolean checkAccess(UserGroupInformation callerUGI, ApplicationAccessType jobOperation);
 
-  VertexLocationHint getVertexLocationHint(TezVertexID vertexId);
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index 5d8b8f7..8d9fbc3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -28,6 +28,7 @@ import org.apache.tez.common.InputSpec;
 import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.DAGPlan.VertexPlan;
 import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
 import org.apache.tez.engine.records.TezTaskID;
 import org.apache.tez.engine.records.TezVertexID;
@@ -39,6 +40,8 @@ import org.apache.tez.engine.records.TezVertexID;
 public interface Vertex extends Comparable<Vertex> {
 
   TezVertexID getVertexId();
+  public VertexPlan getVertexPlan();
+  
   int getDistanceFromRoot();
   String getName();
   VertexState getState();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 0e1206f..2e20692 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.Clock;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
@@ -53,13 +52,15 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGConfiguration;
-import org.apache.tez.dag.api.DAGLocationHint;
+import org.apache.tez.dag.api.DAGPlan.EdgePlan;
+import org.apache.tez.dag.api.DAGPlan.JobPlan;
+import org.apache.tez.dag.api.DAGPlan.VertexPlan;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
+import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
@@ -124,16 +125,14 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   private final long appSubmitTime;
   private final AppContext appContext;
 
-  volatile Map<TezVertexID, Vertex> vertices =
-      new HashMap<TezVertexID, Vertex>();
-  private Map<String, EdgeProperty> edgeProperties = 
-                                          new HashMap<String, EdgeProperty>();
+  volatile Map<TezVertexID, Vertex> vertices = new HashMap<TezVertexID, Vertex>();
+  private Map<String, EdgeProperty> edges = new HashMap<String, EdgeProperty>(); 
   private TezCounters dagCounters = new TezCounters();
   private Object fullCountersLock = new Object();
   private TezCounters fullCounters = null;
 
   public final TezConfiguration conf;
-  public final DAGConfiguration dagPlan;
+  private final JobPlan jobPlan;
 
   //fields initialized in init
   private FileSystem fs;
@@ -151,9 +150,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   private static final DAGSchedulerUpdateTransition 
           DAG_SCHEDULER_UPDATE_TRANSITION = new DAGSchedulerUpdateTransition();
 
-  // Location hints for all vertices in DAG
-  private final DAGLocationHint dagLocationHint;
-
   protected static final
     StateMachineFactory<DAGImpl, DAGState, DAGEventType, DAGEvent>
        stateMachineFactory
@@ -315,7 +311,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
 
   public DAGImpl(TezDAGID dagId, ApplicationAttemptId applicationAttemptId,
       TezConfiguration conf,
-      DAGConfiguration dagPlan,
+      JobPlan jobPlan,
       EventHandler eventHandler,
       TaskAttemptListener taskAttemptListener,
       JobTokenSecretManager jobTokenSecretManager,
@@ -327,14 +323,13 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
       // TODO Recovery
       //List<AMInfo> amInfos,
       TaskHeartbeatHandler thh,
-      AppContext appContext,
-      DAGLocationHint dagLocationHint) {
+      AppContext appContext) {
     this.applicationAttemptId = applicationAttemptId;
     this.dagId = dagId;
-    this.dagPlan = dagPlan;
+    this.jobPlan = jobPlan;
     this.conf = conf;
-    this.dagName = (dagPlan.getName() != null) ? dagPlan.getName() : 
-                                                  "<missing app name>";
+    this.dagName = (jobPlan.getName() != null) ? jobPlan.getName() : "<missing app name>";
+    
     this.userName = appUserName;
     // TODO Metrics
     //this.metrics = metrics;
@@ -357,8 +352,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
 
     this.aclsManager = new ApplicationACLsManager(conf);
 
-    this.dagLocationHint = dagLocationHint;
-
     // This "this leak" is okay because the retained pointer is in an
     //  instance variable.
     stateMachine = stateMachineFactory.make(this);
@@ -380,8 +373,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   }
   
   @Override
-  public DAGConfiguration getDagPlan() {
-    return dagPlan;
+  public JobPlan getJobPlan() {
+    return jobPlan;
   }
 
   EventHandler getEventHandler() {
@@ -812,18 +805,19 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
         */
 
         // create the vertices
-        String[] vertexNames = dag.getDagPlan().getVertices();
-        dag.numVertices = vertexNames.length;
+        dag.numVertices = dag.getJobPlan().getVertexCount();
         for (int i=0; i < dag.numVertices; ++i) {
-          VertexImpl v = createVertex(dag, vertexNames[i], i);
+          String vertexName = dag.getJobPlan().getVertex(i).getName();
+          VertexImpl v = createVertex(dag, vertexName, i);
           dag.addVertex(v);
         }
 
-        dag.edgeProperties = dag.getDagPlan().getEdgeProperties();
-
+        dag.edges = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dag.getJobPlan().getEdgeList());
+        Map<String,EdgePlan> edgePlans = DagTypeConverters.createEdgePlanMapFromDAGPlan(dag.getJobPlan().getEdgeList());
+        
         // setup the dag
         for (Vertex v : dag.vertices.values()) {
-          parseVertexEdges(dag, v);
+          parseVertexEdges(dag, edgePlans, v);
         }
 
         dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
@@ -846,39 +840,43 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
 
     private VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
       TezVertexID vertexId = TezBuilderUtils.newVertexID(dag.getID(), vId);
+      
+      VertexPlan vertexPlan = dag.getJobPlan().getVertex(vId);
+      VertexLocationHint vertexLocationHint = DagTypeConverters.convertFromDAGPlan(vertexPlan.getTaskLocationHintList());
+        
       return new VertexImpl(
-          vertexId, vertexName, dag.conf,
+          vertexId, vertexPlan, vertexName, dag.conf,
           dag.eventHandler, dag.taskAttemptListener,
           dag.jobToken, dag.fsTokens, dag.clock,
           dag.taskHeartbeatHandler, dag.appContext,
-          dag.dagLocationHint.getVertexLocationHint(vertexName));
+          vertexLocationHint);
     }
 
-    private void parseVertexEdges(DAGImpl dag, Vertex vertex) {
-      String[] inVerticesNames =
-          dag.getDagPlan().getInputVertices(vertex.getName());
-      List<String> inEdges =
-          dag.getDagPlan().getInputEdgeIds(vertex.getName());
+    // hooks up this VertexImpl to input and output EdgeProperties
+    private void parseVertexEdges(DAGImpl dag, Map<String, EdgePlan> edgePlans, Vertex vertex) {
+      VertexPlan vertexPlan = vertex.getVertexPlan();
+
       Map<Vertex, EdgeProperty> inVertices =
           new HashMap<Vertex, EdgeProperty>();
-      for (int i=0; i < inVerticesNames.length; ++i) {
-        String vertexName = inVerticesNames[i];
-        inVertices.put(dag.getVertex(vertexName), 
-                       dag.edgeProperties.get(inEdges.get(i)));
-      }
-      vertex.setInputVertices(inVertices);
 
-      String[] outVerticesNames =
-          dag.getDagPlan().getOutputVertices(vertex.getName());
-      List<String> outEdges =
-          dag.getDagPlan().getOutputEdgeIds(vertex.getName());
       Map<Vertex, EdgeProperty> outVertices =
           new HashMap<Vertex, EdgeProperty>();
-      for (int i=0; i < outVerticesNames.length; ++i) {
-        String vertexName = outVerticesNames[i];
-        outVertices.put(dag.getVertex(vertexName), 
-                        dag.edgeProperties.get(outEdges.get(i)));
+
+      for(String inEdgeId : vertexPlan.getInEdgeIdList()){
+        EdgePlan edgePlan = edgePlans.get(inEdgeId);
+        Vertex inVertex = dag.vertexMap.get(edgePlan.getInputVertexName());    
+        EdgeProperty edgeProp = dag.edges.get(inEdgeId);
+        inVertices.put(inVertex, edgeProp);
       }
+      
+      for(String outEdgeId : vertexPlan.getOutEdgeIdList()){
+        EdgePlan edgePlan = edgePlans.get(outEdgeId);
+        Vertex outVertex = dag.vertexMap.get(edgePlan.getOutputVertexName());    
+        EdgeProperty edgeProp = dag.edges.get(outEdgeId);
+        outVertices.put(outVertex, edgeProp);
+      }
+      
+      vertex.setInputVertices(inVertices);
       vertex.setOutputVertices(outVertices);
     }
 
@@ -1145,11 +1143,4 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
       job.finished(DAGState.ERROR);
     }
   }
-
-  @Override
-  public VertexLocationHint getVertexLocationHint(TezVertexID vertexId) {
-    return dagLocationHint.getVertexLocationHint(
-        getVertex(vertexId).getName());
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 80de958..a423db1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -55,7 +55,8 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.tez.common.InputSpec;
 import org.apache.tez.common.OutputSpec;
 import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGConfiguration;
+import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
 import org.apache.tez.dag.api.EdgeProperty;
@@ -324,7 +325,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   private Credentials fsTokens;
   private Token<JobTokenIdentifier> jobToken;
 
-  private final TezVertexID vertexId;
+  private final TezVertexID vertexId;  //runtime assigned id.
+  private final VertexPlan vertexPlan;  
 
   private final String vertexName;
   private final String processorName;
@@ -341,7 +343,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   private Map<String, String> environment;
   private String javaOpts;
   
-  public VertexImpl(TezVertexID vertexId, String vertexName,
+  public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, String vertexName,
       TezConfiguration conf, EventHandler eventHandler,
       TaskAttemptListener taskAttemptListener,
       Token<JobTokenIdentifier> jobToken,
@@ -353,6 +355,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       TaskHeartbeatHandler thh,
       AppContext appContext, VertexLocationHint vertexLocationHint) {
     this.vertexId = vertexId;
+    this.vertexPlan = vertexPlan;
     this.vertexName = vertexName;
     this.conf = conf;
     //this.metrics = metrics;
@@ -373,12 +376,11 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     this.committer = new NullVertexOutputCommitter();
     this.vertexLocationHint = vertexLocationHint;
     
-    this.taskResource = getDAGPlan().getVertexResource(getName());
-    this.processorName = getDAGPlan().getVertexTaskModuleClassName(getName());
-    this.localResources = getDAGPlan().getVertexLocalResources(getName());
-    this.environment = getDAGPlan().getVertexEnv(getName());
-    
-    this.javaOpts = getDAGPlan().getVertexJavaOpts(getName());
+    this.taskResource = DagTypeConverters.CreateResourceRequestFromTaskConfig(vertexPlan.getTaskConfig());
+    this.processorName = vertexPlan.hasProcessorName() ? vertexPlan.getProcessorName() : null;  
+    this.localResources = DagTypeConverters.createLocalResourceMapFromDAGPlan(vertexPlan.getTaskConfig().getLocalResourceList());
+    this.environment = DagTypeConverters.createEnvironmentMapFromDAGPlan(vertexPlan.getTaskConfig().getEnvironmentSettingList());
+    this.javaOpts = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan.getTaskConfig().getJavaOpts() : null;
     
     // This "this leak" is okay because the retained pointer is in an
     //  instance variable.
@@ -395,6 +397,11 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   }
   
   @Override
+  public VertexPlan getVertexPlan() {
+    return vertexPlan;
+  }
+  
+  @Override
   public int getDistanceFromRoot() {
     return distanceFromRoot;
   }
@@ -727,7 +734,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
 
         // TODODAGAM
         // TODO: Splits?
-        vertex.numTasks = vertex.getDAGPlan().getNumVertexTasks(vertex.getName());
+        
+        vertex.numTasks = vertex.getVertexPlan().getTaskConfig().getNumTasks();
+        
         /*
         TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(job, job.jobId);
         job.numMapTasks = taskSplitMetaInfo.length;
@@ -1217,10 +1226,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     return appContext.getDAG();
   }
   
-  DAGConfiguration getDAGPlan() {
-    return getDAG().getDagPlan();
-  }
-
   // TODO Eventually remove synchronization.
   @Override
   public synchronized List<InputSpec> getInputSpecList() {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
index 331e2a3..b3db770 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
@@ -82,6 +82,12 @@ public class TezEngineChildJVM {
     //set custom javaOpts
     vargs.add(javaOpts); 
     
+//[Debug Task] Current simplest way to attach debugger to  Tez Child Task
+// Uncomment the following, then launch a regular job
+// Works best on one-box configured with a single container (hence one task at a time). 
+//    LOG.error(" !!!!!!!!! Launching Child-Task in debug/suspend mode.  Attach to port 8003 !!!!!!!!");
+//    vargs.add("-Xdebug -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=8003,server=y,suspend=y");
+    
     Path childTmpDir = new Path(Environment.PWD.$(),
         YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
     vargs.add("-Djava.io.tmpdir=" + childTmpDir);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/test/java/org/apache/tez/dag/api/TestDAGLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/api/TestDAGLocationHint.java b/tez-dag/src/test/java/org/apache/tez/dag/api/TestDAGLocationHint.java
deleted file mode 100644
index 3deaee7..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/api/TestDAGLocationHint.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestDAGLocationHint {
-
-  private DataInput in;
-  private DataOutput out;
-  private ByteArrayOutputStream bOut;
-
-  @Before
-  public void setup() {
-    bOut = new ByteArrayOutputStream();
-    out = new DataOutputStream(bOut);
-  }
-
-  @After
-  public void teardown() {
-    in = null;
-    out = null;
-    bOut = null;
-  }
-
-  @Test
-  public void testNullDAGLocationHintSerDes() throws IOException {
-    DAGLocationHint expected = new DAGLocationHint();
-    expected.write(out);
-    in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
-    DAGLocationHint actual = new DAGLocationHint();
-    actual.readFields(in);
-    Assert.assertNotNull(actual.getVertexLocationHints());
-    Assert.assertEquals(0, actual.getVertexLocationHints().size());
-  }
-
-  @Test
-  public void testDAGLocationHintSerDes() throws IOException {
-    String[] hosts = { "h1", "h2", "", null };
-    String[] racks = { "r1", "r2" };
-
-    VertexLocationHint vertexLocationHint = new VertexLocationHint(4);
-    vertexLocationHint.getTaskLocationHints()[0] =
-        new TaskLocationHint(hosts, racks);
-    DAGLocationHint expected = new DAGLocationHint();
-    expected.getVertexLocationHints().put("v1", null);
-    expected.getVertexLocationHints().put("v2", new VertexLocationHint());
-    expected.getVertexLocationHints().put("v3", vertexLocationHint);
-    expected.write(out);
-
-    in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
-    DAGLocationHint actual = new DAGLocationHint();
-    actual.readFields(in);
-    Assert.assertNotNull(actual.getVertexLocationHints());
-    Assert.assertEquals(3, actual.getVertexLocationHints().size());
-
-    Assert.assertNull(actual.getVertexLocationHint("v1"));
-    Assert.assertNotNull(actual.getVertexLocationHint("v2"));
-    Assert.assertNotNull(actual.getVertexLocationHint("v3"));
-
-    Assert.assertEquals(0, actual.getVertexLocationHint("v2").getNumTasks());
-    Assert.assertEquals(0,
-        actual.getVertexLocationHint("v2").getTaskLocationHints().length);
-
-    Assert.assertEquals(4, actual.getVertexLocationHint("v3").getNumTasks());
-    Assert.assertEquals(4,
-        actual.getVertexLocationHint("v3").getTaskLocationHints().length);
-    Assert.assertNotNull(
-        actual.getVertexLocationHint("v3").getTaskLocationHints()[0]);
-    Assert.assertArrayEquals(racks,
-        actual.getVertexLocationHint("v3").getTaskLocationHints()[0].
-            getRacks());
-    Assert.assertArrayEquals(hosts,
-        actual.getVertexLocationHint("v3").getTaskLocationHints()[0].
-            getDataLocalHosts());
-    Assert.assertNull(
-        actual.getVertexLocationHint("v3").getTaskLocationHints()[1]);
-    Assert.assertNull(
-        actual.getVertexLocationHint("v3").getTaskLocationHints()[2]);
-    Assert.assertNull(
-        actual.getVertexLocationHint("v3").getTaskLocationHints()[3]);
-  }
-
-
-}
-


[31/50] [abbrv] git commit: TEZ-137 VertexHistoryEvents are using VertexStatus.state instead of VertexImpl.state (bikas)

Posted by ss...@apache.org.
TEZ-137 VertexHistoryEvents are using VertexStatus.state instead of VertexImpl.state (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/3bb954bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/3bb954bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/3bb954bd

Branch: refs/heads/master
Commit: 3bb954bd9153f1b6b7e394776d9169cd99acde1b
Parents: a2a7261
Author: Bikas Saha <bi...@apache.org>
Authored: Tue May 28 13:33:52 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue May 28 13:33:52 2013 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/tez/client/TezClient.java |    3 -
 .../hadoop/mapred/MRVertexOutputCommitter.java     |   19 +++++-
 .../apache/tez/dag/api/committer/DAGStatus.java    |   39 -----------
 .../api/committer/NullVertexOutputCommitter.java   |    4 +-
 .../dag/api/committer/VertexOutputCommitter.java   |    1 +
 .../apache/tez/dag/api/committer/VertexStatus.java |   51 ---------------
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |    2 +-
 .../apache/tez/dag/app/dag/impl/VertexImpl.java    |   12 ++--
 .../tez/dag/history/events/DAGFinishedEvent.java   |    2 +-
 .../dag/history/events/VertexFinishedEvent.java    |    2 +-
 .../tez/dag/app/dag/impl/TestVertexImpl.java       |    4 +-
 11 files changed, 32 insertions(+), 107 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
index f794014..73c97b3 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezRemoteException;
 import org.apache.tez.dag.api.client.DAGClient;
 import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.api.client.VertexStatus;
@@ -45,7 +44,6 @@ public class TezClient {
 
   public DAGClient getDAGClient(String appIdStr) throws IOException, TezException {
     try {
-      System.out.println("Fetching app: " + appIdStr);
       ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
       YarnClient yarnClient = new YarnClientImpl();
       yarnClient.init(conf);
@@ -60,7 +58,6 @@ public class TezClient {
   }
   
   public DAGClient getDAGClient(String host, int port) throws IOException {
-    System.out.println("App host port: " + host + ":" + port);
     InetSocketAddress addr = new InetSocketAddress(host, port);
     DAGClient dagClient;
     dagClient = new DAGClientRPCImpl(1, addr, conf);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
index 7c886d6..171c244 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
@@ -33,9 +33,10 @@ import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.YarnException;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.client.VertexStatus;
 import org.apache.tez.dag.api.committer.VertexContext;
 import org.apache.tez.dag.api.committer.VertexOutputCommitter;
-import org.apache.tez.dag.api.committer.VertexStatus;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
 import org.apache.tez.dag.utils.TezBuilderUtils;
@@ -102,7 +103,21 @@ public class MRVertexOutputCommitter extends VertexOutputCommitter {
   }
 
   private State getJobStateFromVertexStatusState(VertexStatus.State state) {
-    return JobStatus.State.valueOf(state.name());
+    switch(state) {
+    case INITED:
+      return JobStatus.State.PREP;
+    case RUNNING:
+      return JobStatus.State.RUNNING;
+    case SUCCEEDED:
+      return JobStatus.State.SUCCEEDED;
+    case KILLED:
+      return JobStatus.State.KILLED;
+    case FAILED:
+    case ERROR:
+      return JobStatus.State.FAILED;
+    default:
+      throw new TezException("Unknown VertexStatus.State: " + state);
+    }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
deleted file mode 100644
index 587e18f..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.committer;
-
-public class DAGStatus {
-  public static enum State {
-    RUNNING(1),
-    SUCCEEDED(2),
-    FAILED(3),
-    PREP(4),
-    KILLED(5);
-    
-    int value;
-    
-    State(int value) {
-      this.value = value;
-    }
-    
-    public int getValue() {
-      return value; 
-    }
-    
-  };
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
index c284ab5..08b2d4a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.api.committer;
 
 import java.io.IOException;
 
-import org.apache.tez.dag.api.committer.VertexStatus.State;
+import org.apache.tez.dag.api.client.VertexStatus;
 
 public class NullVertexOutputCommitter extends VertexOutputCommitter {
 
@@ -40,7 +40,7 @@ public class NullVertexOutputCommitter extends VertexOutputCommitter {
   }
 
   @Override
-  public void abortVertex(State finalState) throws IOException {
+  public void abortVertex(VertexStatus.State finalState) throws IOException {
     // Nothing to do
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
index b814d0e..34c6e89 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.tez.dag.api.client.VertexStatus;
 
 @Public
 @Unstable

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
deleted file mode 100644
index a78027a..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.committer;
-
-public class VertexStatus {
-
-  public static enum State {
-    RUNNING(1),
-    SUCCEEDED(2),
-    FAILED(3),
-    PREP(4), // TODO change to INITING
-    KILLED(5);
-
-    int value;
-
-    State(int value) {
-      this.value = value;
-    }
-
-    public int getValue() {
-      return value;
-    }
-
-  };
-
-  private State runState;
-
-  public synchronized State getState() {
-    return runState;
-  }
-
-  public synchronized void setState(State state) {
-    this.runState = state;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 4c63d8f..cc6699d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -59,10 +59,10 @@ import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.VertexLocationHint;
+import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.api.client.DAGStatusBuilder;
 import org.apache.tez.dag.api.client.ProgressBuilder;
 import org.apache.tez.dag.api.client.VertexStatusBuilder;
-import org.apache.tez.dag.api.committer.DAGStatus;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index ce4c609..e34382f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -61,15 +61,15 @@ import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
 import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
 import org.apache.tez.dag.api.client.ProgressBuilder;
+import org.apache.tez.dag.api.client.VertexStatus;
 import org.apache.tez.dag.api.client.VertexStatusBuilder;
 import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
 import org.apache.tez.dag.api.committer.VertexContext;
 import org.apache.tez.dag.api.committer.VertexOutputCommitter;
-import org.apache.tez.dag.api.committer.VertexStatus;
-import org.apache.tez.dag.api.committer.VertexStatus.State;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
@@ -744,23 +744,25 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       case KILL_WAIT:
         eventHandler.handle(new DAGEventVertexCompleted(getVertexId(),
             finalState));
-        logJobHistoryVertexFailedEvent(State.KILLED);
+        logJobHistoryVertexFailedEvent(VertexStatus.State.KILLED);
         break;
       case ERROR:
         eventHandler.handle(new DAGEvent(getDAGId(),
             DAGEventType.INTERNAL_ERROR));
-        logJobHistoryVertexFailedEvent(State.FAILED);
+        logJobHistoryVertexFailedEvent(VertexStatus.State.FAILED);
         break;
       case FAILED:
         eventHandler.handle(new DAGEventVertexCompleted(getVertexId(),
             finalState));
-        logJobHistoryVertexFailedEvent(State.FAILED);
+        logJobHistoryVertexFailedEvent(VertexStatus.State.FAILED);
         break;
       case SUCCEEDED:
         eventHandler.handle(new DAGEventVertexCompleted(getVertexId(),
             finalState));
         logJobHistoryVertexFinishedEvent();
         break;
+      default:
+        throw new TezException("Unexpected VertexState: " + finalState);
     }
     return finalState;
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
index 5dea9a7..8794cea 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.history.events;
 
-import org.apache.tez.dag.api.committer.DAGStatus;
+import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.DAGFinished;
 import org.apache.tez.dag.history.avro.HistoryEventType;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
index 91c209f..7e86c66 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
@@ -18,7 +18,7 @@
 
 package org.apache.tez.dag.history.events;
 
-import org.apache.tez.dag.api.committer.VertexStatus;
+import org.apache.tez.dag.api.client.VertexStatus;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.avro.HistoryEventType;
 import org.apache.tez.dag.history.avro.VertexFinished;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index f78a7b8..e4a1040 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -50,10 +50,10 @@ import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint;
+import org.apache.tez.dag.api.client.VertexStatus;
 import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
 import org.apache.tez.dag.api.committer.VertexContext;
 import org.apache.tez.dag.api.committer.VertexOutputCommitter;
-import org.apache.tez.dag.api.committer.VertexStatus.State;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.TaskAttemptListener;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
@@ -147,7 +147,7 @@ public class TestVertexImpl {
     }
 
     @Override
-    public void abortVertex(State finalState) throws IOException {
+    public void abortVertex(VertexStatus.State finalState) throws IOException {
       ++abortCounter;
       if (throwErrorOnAbort) {
         throw new IOException("I can throwz exceptions in abort");