You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2015/08/26 01:48:48 UTC

[1/7] tez git commit: TEZ-2708. Rename classes and variables post TEZ-2003 changes. (sseth)

Repository: tez
Updated Branches:
  refs/heads/master dc0ee0115 -> 8b278ea84


http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
new file mode 100644
index 0000000..98b7baa
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
@@ -0,0 +1,708 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.ContainerSignatureMatcher;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TaskLocationHint;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.client.DAGClientServer;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.ContainerContext;
+import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
+import org.apache.tez.dag.app.dag.TaskAttempt;
+import org.apache.tez.dag.app.dag.impl.TaskAttemptImpl;
+import org.apache.tez.dag.app.dag.impl.TaskImpl;
+import org.apache.tez.dag.app.dag.impl.VertexImpl;
+import org.apache.tez.dag.app.rm.container.AMContainer;
+import org.apache.tez.dag.app.rm.container.AMContainerEventAssignTA;
+import org.apache.tez.dag.app.rm.container.AMContainerEventCompleted;
+import org.apache.tez.dag.app.rm.container.AMContainerEventType;
+import org.apache.tez.dag.app.rm.container.AMContainerMap;
+import org.apache.tez.dag.app.rm.container.AMContainerState;
+import org.apache.tez.dag.app.web.WebUIService;
+import org.apache.tez.dag.records.TaskAttemptTerminationCause;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.serviceplugins.api.TaskScheduler;
+import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+@SuppressWarnings("rawtypes")
+public class TestTaskSchedulerManager {
+  
+  class TestEventHandler implements EventHandler{
+    List<Event> events = Lists.newLinkedList();
+    @Override
+    public void handle(Event event) {
+      events.add(event);
+    }
+  }
+  
+  class MockTaskSchedulerManager extends TaskSchedulerManager {
+
+    final AtomicBoolean notify = new AtomicBoolean(false);
+    
+    public MockTaskSchedulerManager(AppContext appContext,
+                                    DAGClientServer clientService, EventHandler eventHandler,
+                                    ContainerSignatureMatcher containerSignatureMatcher,
+                                    WebUIService webUI) {
+      super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI,
+          Lists.newArrayList(new NamedEntityDescriptor("FakeDescriptor", null)), false);
+    }
+
+    @Override
+    protected void instantiateSchedulers(String host, int port, String trackingUrl,
+                                         AppContext appContext) {
+      taskSchedulers[0] = mockTaskScheduler;
+      taskSchedulerServiceWrappers[0] = new ServicePluginLifecycleAbstractService<>(taskSchedulers[0]);
+    }
+    
+    @Override
+    protected void notifyForTest() {
+      synchronized (notify) {
+        notify.set(true);
+        notify.notifyAll();
+      }
+    }
+    
+  }
+
+  AppContext mockAppContext;
+  DAGClientServer mockClientService;
+  TestEventHandler mockEventHandler;
+  ContainerSignatureMatcher mockSigMatcher;
+  MockTaskSchedulerManager schedulerHandler;
+  TaskScheduler mockTaskScheduler;
+  AMContainerMap mockAMContainerMap;
+  WebUIService mockWebUIService;
+
+  @Before
+  public void setup() {
+    mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
+    doReturn(new Configuration(false)).when(mockAppContext).getAMConf();
+    mockClientService = mock(DAGClientServer.class);
+    mockEventHandler = new TestEventHandler();
+    mockSigMatcher = mock(ContainerSignatureMatcher.class);
+    mockTaskScheduler = mock(TaskScheduler.class);
+    mockAMContainerMap = mock(AMContainerMap.class);
+    mockWebUIService = mock(WebUIService.class);
+    when(mockAppContext.getAllContainers()).thenReturn(mockAMContainerMap);
+    when(mockClientService.getBindAddress()).thenReturn(new InetSocketAddress(10000));
+    schedulerHandler = new MockTaskSchedulerManager(
+        mockAppContext, mockClientService, mockEventHandler, mockSigMatcher, mockWebUIService);
+  }
+
+  @Test(timeout = 5000)
+  public void testSimpleAllocate() throws Exception {
+    Configuration conf = new Configuration(false);
+    schedulerHandler.init(conf);
+    schedulerHandler.start();
+
+    TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
+    TezTaskAttemptID mockAttemptId = mock(TezTaskAttemptID.class);
+    when(mockAttemptId.getId()).thenReturn(0);
+    when(mockTaskAttempt.getID()).thenReturn(mockAttemptId);
+    Resource resource = Resource.newInstance(1024, 1);
+    ContainerContext containerContext =
+        new ContainerContext(new HashMap<String, LocalResource>(), new Credentials(),
+            new HashMap<String, String>(), "");
+    int priority = 10;
+    TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(), null);
+
+    ContainerId mockCId = mock(ContainerId.class);
+    Container container = mock(Container.class);
+    when(container.getId()).thenReturn(mockCId);
+
+    AMContainer mockAMContainer = mock(AMContainer.class);
+    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
+    when(mockAMContainer.getState()).thenReturn(AMContainerState.IDLE);
+
+    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
+
+    AMSchedulerEventTALaunchRequest lr =
+        new AMSchedulerEventTALaunchRequest(mockAttemptId, resource, null, mockTaskAttempt, locHint,
+            priority, containerContext, 0, 0, 0);
+    schedulerHandler.taskAllocated(0, mockTaskAttempt, lr, container);
+    assertEquals(2, mockEventHandler.events.size());
+    assertTrue(mockEventHandler.events.get(1) instanceof AMContainerEventAssignTA);
+    AMContainerEventAssignTA assignEvent =
+        (AMContainerEventAssignTA) mockEventHandler.events.get(1);
+    assertEquals(priority, assignEvent.getPriority());
+    assertEquals(mockAttemptId, assignEvent.getTaskAttemptId());
+  }
+
+  @Test (timeout = 5000)
+  public void testTaskBasedAffinity() throws Exception {
+    Configuration conf = new Configuration(false);
+    schedulerHandler.init(conf);
+    schedulerHandler.start();
+
+    TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
+    TezTaskAttemptID taId = mock(TezTaskAttemptID.class);
+    String affVertexName = "srcVertex";
+    int affTaskIndex = 1;
+    TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(affVertexName, affTaskIndex);
+    VertexImpl affVertex = mock(VertexImpl.class);
+    TaskImpl affTask = mock(TaskImpl.class);
+    TaskAttemptImpl affAttempt = mock(TaskAttemptImpl.class);
+    ContainerId affCId = mock(ContainerId.class);
+    when(affVertex.getTotalTasks()).thenReturn(2);
+    when(affVertex.getTask(affTaskIndex)).thenReturn(affTask);
+    when(affTask.getSuccessfulAttempt()).thenReturn(affAttempt);
+    when(affAttempt.getAssignedContainerID()).thenReturn(affCId);
+    when(mockAppContext.getCurrentDAG().getVertex(affVertexName)).thenReturn(affVertex);
+    Resource resource = Resource.newInstance(100, 1);
+    AMSchedulerEventTALaunchRequest event = new AMSchedulerEventTALaunchRequest
+        (taId, resource, null, mockTaskAttempt, locHint, 3, null, 0, 0, 0);
+    schedulerHandler.notify.set(false);
+    schedulerHandler.handle(event);
+    synchronized (schedulerHandler.notify) {
+      while (!schedulerHandler.notify.get()) {
+        schedulerHandler.notify.wait();
+      }
+    }
+    
+    // verify mockTaskAttempt affinitized to expected affCId
+    verify(mockTaskScheduler, times(1)).allocateTask(mockTaskAttempt, resource, affCId,
+        Priority.newInstance(3), null, event);
+    
+    schedulerHandler.stop();
+    schedulerHandler.close();
+  }
+  
+  @Test (timeout = 5000)
+  public void testContainerPreempted() throws IOException {
+    Configuration conf = new Configuration(false);
+    schedulerHandler.init(conf);
+    schedulerHandler.start();
+    
+    String diagnostics = "Container preempted by RM.";
+    TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
+    ContainerStatus mockStatus = mock(ContainerStatus.class);
+    ContainerId mockCId = mock(ContainerId.class);
+    AMContainer mockAMContainer = mock(AMContainer.class);
+    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
+    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
+    when(mockStatus.getContainerId()).thenReturn(mockCId);
+    when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
+    when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.PREEMPTED);
+    schedulerHandler.containerCompleted(0, mockTask, mockStatus);
+    assertEquals(1, mockEventHandler.events.size());
+    Event event = mockEventHandler.events.get(0);
+    assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
+    AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
+    assertEquals(mockCId, completedEvent.getContainerId());
+    assertEquals("Container preempted externally. Container preempted by RM.",
+        completedEvent.getDiagnostics());
+    assertTrue(completedEvent.isPreempted());
+    assertEquals(TaskAttemptTerminationCause.EXTERNAL_PREEMPTION,
+        completedEvent.getTerminationCause());
+    Assert.assertFalse(completedEvent.isDiskFailed());
+
+    schedulerHandler.stop();
+    schedulerHandler.close();
+  }
+  
+  @Test (timeout = 5000)
+  public void testContainerInternalPreempted() throws IOException {
+    Configuration conf = new Configuration(false);
+    schedulerHandler.init(conf);
+    schedulerHandler.start();
+
+    AMContainer mockAmContainer = mock(AMContainer.class);
+    when(mockAmContainer.getTaskSchedulerIdentifier()).thenReturn(0);
+    when(mockAmContainer.getContainerLauncherIdentifier()).thenReturn(0);
+    when(mockAmContainer.getTaskCommunicatorIdentifier()).thenReturn(0);
+    ContainerId mockCId = mock(ContainerId.class);
+    verify(mockTaskScheduler, times(0)).deallocateContainer((ContainerId) any());
+    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAmContainer);
+    schedulerHandler.preemptContainer(0, mockCId);
+    verify(mockTaskScheduler, times(1)).deallocateContainer(mockCId);
+    assertEquals(1, mockEventHandler.events.size());
+    Event event = mockEventHandler.events.get(0);
+    assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
+    AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
+    assertEquals(mockCId, completedEvent.getContainerId());
+    assertEquals("Container preempted internally", completedEvent.getDiagnostics());
+    assertTrue(completedEvent.isPreempted());
+    Assert.assertFalse(completedEvent.isDiskFailed());
+    assertEquals(TaskAttemptTerminationCause.INTERNAL_PREEMPTION,
+        completedEvent.getTerminationCause());
+
+    schedulerHandler.stop();
+    schedulerHandler.close();
+  }
+  
+  @Test (timeout = 5000)
+  public void testContainerDiskFailed() throws IOException {
+    Configuration conf = new Configuration(false);
+    schedulerHandler.init(conf);
+    schedulerHandler.start();
+    
+    String diagnostics = "NM disk failed.";
+    TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
+    ContainerStatus mockStatus = mock(ContainerStatus.class);
+    ContainerId mockCId = mock(ContainerId.class);
+    AMContainer mockAMContainer = mock(AMContainer.class);
+    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
+    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
+    when(mockStatus.getContainerId()).thenReturn(mockCId);
+    when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
+    when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.DISKS_FAILED);
+    schedulerHandler.containerCompleted(0, mockTask, mockStatus);
+    assertEquals(1, mockEventHandler.events.size());
+    Event event = mockEventHandler.events.get(0);
+    assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
+    AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
+    assertEquals(mockCId, completedEvent.getContainerId());
+    assertEquals("Container disk failed. NM disk failed.",
+        completedEvent.getDiagnostics());
+    Assert.assertFalse(completedEvent.isPreempted());
+    assertTrue(completedEvent.isDiskFailed());
+    assertEquals(TaskAttemptTerminationCause.NODE_DISK_ERROR,
+        completedEvent.getTerminationCause());
+
+    schedulerHandler.stop();
+    schedulerHandler.close();
+  }
+
+  @Test (timeout = 5000)
+  public void testContainerExceededPMem() throws IOException {
+    Configuration conf = new Configuration(false);
+    schedulerHandler.init(conf);
+    schedulerHandler.start();
+
+    String diagnostics = "Exceeded Physical Memory";
+    TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
+    ContainerStatus mockStatus = mock(ContainerStatus.class);
+    ContainerId mockCId = mock(ContainerId.class);
+    AMContainer mockAMContainer = mock(AMContainer.class);
+    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
+    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
+    when(mockStatus.getContainerId()).thenReturn(mockCId);
+    when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
+    // use -104 rather than ContainerExitStatus.KILLED_EXCEEDED_PMEM because
+    // ContainerExitStatus.KILLED_EXCEEDED_PMEM is only available after hadoop-2.5
+    when(mockStatus.getExitStatus()).thenReturn(-104);
+    schedulerHandler.containerCompleted(0, mockTask, mockStatus);
+    assertEquals(1, mockEventHandler.events.size());
+    Event event = mockEventHandler.events.get(0);
+    assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
+    AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
+    assertEquals(mockCId, completedEvent.getContainerId());
+    assertEquals("Container failed, exitCode=-104. Exceeded Physical Memory",
+        completedEvent.getDiagnostics());
+    Assert.assertFalse(completedEvent.isPreempted());
+    Assert.assertFalse(completedEvent.isDiskFailed());
+    assertEquals(TaskAttemptTerminationCause.CONTAINER_EXITED,
+        completedEvent.getTerminationCause());
+
+    schedulerHandler.stop();
+    schedulerHandler.close();
+  }
+
+  @Test (timeout = 5000)
+  public void testHistoryUrlConf() throws Exception {
+    Configuration conf = schedulerHandler.appContext.getAMConf();
+
+    // ensure history url is empty when timeline server is not the logging class
+    conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9999");
+    assertTrue("".equals(schedulerHandler.getHistoryUrl()));
+
+    // ensure expansion of url happens
+    conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
+        "org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService");
+    final ApplicationId mockApplicationId = mock(ApplicationId.class);
+    doReturn("TEST_APP_ID").when(mockApplicationId).toString();
+    doReturn(mockApplicationId).when(mockAppContext).getApplicationID();
+    assertTrue("http://ui-host:9999/#/tez-app/TEST_APP_ID"
+        .equals(schedulerHandler.getHistoryUrl()));
+
+    // ensure the trailing / in history url is handled
+    conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9998/");
+    assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
+        .equals(schedulerHandler.getHistoryUrl()));
+
+    // ensure missing scheme in history url is handled
+    conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/");
+    Assert.assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
+        .equals(schedulerHandler.getHistoryUrl()));
+
+    // handle bad template ex without begining /
+    conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
+        "__HISTORY_URL_BASE__#/somepath");
+    assertTrue("http://ui-host:9998/#/somepath"
+        .equals(schedulerHandler.getHistoryUrl()));
+
+    conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
+        "__HISTORY_URL_BASE__?viewPath=tez-app/__APPLICATION_ID__");
+    conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://localhost/ui/tez");
+    assertTrue("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID"
+        .equals(schedulerHandler.getHistoryUrl()));
+
+  }
+
+  @Test(timeout = 5000)
+  public void testNoSchedulerSpecified() throws IOException {
+    try {
+      new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
+          mockSigMatcher, mockWebUIService, null, false);
+      fail("Expecting an IllegalStateException with no schedulers specified");
+    } catch (IllegalArgumentException e) {
+    }
+  }
+
+  // Verified via statics
+  @Test(timeout = 5000)
+  public void testCustomTaskSchedulerSetup() throws IOException {
+    Configuration conf = new Configuration(false);
+    conf.set("testkey", "testval");
+    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
+
+    String customSchedulerName = "fakeScheduler";
+    List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
+    ByteBuffer bb = ByteBuffer.allocate(4);
+    bb.putInt(0, 3);
+    UserPayload userPayload = UserPayload.create(bb);
+    taskSchedulers.add(
+        new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName())
+            .setUserPayload(userPayload));
+    taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+        .setUserPayload(defaultPayload));
+
+    TSEHForMultipleSchedulersTest tseh =
+        new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
+            mockSigMatcher, mockWebUIService, taskSchedulers, false);
+
+    tseh.init(conf);
+    tseh.start();
+
+    // Verify that the YARN task scheduler is installed by default
+    assertTrue(tseh.getYarnSchedulerCreated());
+    assertFalse(tseh.getUberSchedulerCreated());
+    assertEquals(2, tseh.getNumCreateInvocations());
+
+    // Verify the order of the schedulers
+    assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
+    assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));
+
+    // Verify the payload setup for the custom task scheduler
+    assertNotNull(tseh.getTaskSchedulerContext(0));
+    assertEquals(bb, tseh.getTaskSchedulerContext(0).getInitialUserPayload().getPayload());
+
+    // Verify the payload on the yarn scheduler
+    assertNotNull(tseh.getTaskSchedulerContext(1));
+    Configuration parsed = TezUtils.createConfFromUserPayload(tseh.getTaskSchedulerContext(1).getInitialUserPayload());
+    assertEquals("testval", parsed.get("testkey"));
+  }
+
+  @Test(timeout = 5000)
+  public void testTaskSchedulerRouting() throws Exception {
+    Configuration conf = new Configuration(false);
+    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
+
+    String customSchedulerName = "fakeScheduler";
+    List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
+    ByteBuffer bb = ByteBuffer.allocate(4);
+    bb.putInt(0, 3);
+    UserPayload userPayload = UserPayload.create(bb);
+    taskSchedulers.add(
+        new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName())
+            .setUserPayload(userPayload));
+    taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+        .setUserPayload(defaultPayload));
+
+    TSEHForMultipleSchedulersTest tseh =
+        new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
+            mockSigMatcher, mockWebUIService, taskSchedulers, false);
+
+    tseh.init(conf);
+    tseh.start();
+
+    // Verify that the YARN task scheduler is installed by default
+    assertTrue(tseh.getYarnSchedulerCreated());
+    assertFalse(tseh.getUberSchedulerCreated());
+    assertEquals(2, tseh.getNumCreateInvocations());
+
+    // Verify the order of the schedulers
+    assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
+    assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));
+
+    verify(tseh.getTestTaskScheduler(0)).initialize();
+    verify(tseh.getTestTaskScheduler(0)).start();
+
+    ApplicationId appId = ApplicationId.newInstance(1000, 1);
+    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
+    TezVertexID vertexID = TezVertexID.getInstance(dagId, 1);
+    TezTaskID taskId1 = TezTaskID.getInstance(vertexID, 1);
+    TezTaskAttemptID attemptId11 = TezTaskAttemptID.getInstance(taskId1, 1);
+    TezTaskID taskId2 = TezTaskID.getInstance(vertexID, 2);
+    TezTaskAttemptID attemptId21 = TezTaskAttemptID.getInstance(taskId2, 1);
+
+    Resource resource = Resource.newInstance(1024, 1);
+
+    TaskAttempt mockTaskAttempt1 = mock(TaskAttempt.class);
+    TaskAttempt mockTaskAttempt2 = mock(TaskAttempt.class);
+
+    AMSchedulerEventTALaunchRequest launchRequest1 =
+        new AMSchedulerEventTALaunchRequest(attemptId11, resource, mock(TaskSpec.class),
+            mockTaskAttempt1, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 0, 0,
+            0);
+
+    tseh.handle(launchRequest1);
+
+    verify(tseh.getTestTaskScheduler(0)).allocateTask(eq(mockTaskAttempt1), eq(resource),
+        any(String[].class), any(String[].class), any(Priority.class), any(Object.class),
+        eq(launchRequest1));
+
+    AMSchedulerEventTALaunchRequest launchRequest2 =
+        new AMSchedulerEventTALaunchRequest(attemptId21, resource, mock(TaskSpec.class),
+            mockTaskAttempt2, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 1, 0,
+            0);
+    tseh.handle(launchRequest2);
+    verify(tseh.getTestTaskScheduler(1)).allocateTask(eq(mockTaskAttempt2), eq(resource),
+        any(String[].class), any(String[].class), any(Priority.class), any(Object.class),
+        eq(launchRequest2));
+  }
+
+  private static class TSEHForMultipleSchedulersTest extends TaskSchedulerManager {
+
+    private final TaskScheduler yarnTaskScheduler;
+    private final TaskScheduler uberTaskScheduler;
+    private final AtomicBoolean uberSchedulerCreated = new AtomicBoolean(false);
+    private final AtomicBoolean yarnSchedulerCreated = new AtomicBoolean(false);
+    private final AtomicInteger numCreateInvocations = new AtomicInteger(0);
+    private final Set<Integer> seenSchedulers = new HashSet<>();
+    private final List<TaskSchedulerContext> taskSchedulerContexts = new LinkedList<>();
+    private final List<String> taskSchedulerNames = new LinkedList<>();
+    private final List<TaskScheduler> testTaskSchedulers = new LinkedList<>();
+
+    public TSEHForMultipleSchedulersTest(AppContext appContext,
+                                         DAGClientServer clientService,
+                                         EventHandler eventHandler,
+                                         ContainerSignatureMatcher containerSignatureMatcher,
+                                         WebUIService webUI,
+                                         List<NamedEntityDescriptor> schedulerDescriptors,
+                                         boolean isPureLocalMode) {
+      super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI,
+          schedulerDescriptors, isPureLocalMode);
+      yarnTaskScheduler = mock(TaskScheduler.class);
+      uberTaskScheduler = mock(TaskScheduler.class);
+    }
+
+    @Override
+    TaskScheduler createTaskScheduler(String host, int port, String trackingUrl,
+                                      AppContext appContext,
+                                      NamedEntityDescriptor taskSchedulerDescriptor,
+                                      long customAppIdIdentifier,
+                                      int schedulerId) {
+
+      numCreateInvocations.incrementAndGet();
+      boolean added = seenSchedulers.add(schedulerId);
+      assertTrue("Cannot add multiple schedulers with the same schedulerId", added);
+      taskSchedulerNames.add(taskSchedulerDescriptor.getEntityName());
+      return super.createTaskScheduler(host, port, trackingUrl, appContext, taskSchedulerDescriptor,
+          customAppIdIdentifier, schedulerId);
+    }
+
+    @Override
+    TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
+      // Avoid wrapping in threads
+      return rawContext;
+    }
+
+    @Override
+    TaskScheduler createYarnTaskScheduler(TaskSchedulerContext taskSchedulerContext, int schedulerId) {
+      taskSchedulerContexts.add(taskSchedulerContext);
+      testTaskSchedulers.add(yarnTaskScheduler);
+      yarnSchedulerCreated.set(true);
+      return yarnTaskScheduler;
+    }
+
+    @Override
+    TaskScheduler createUberTaskScheduler(TaskSchedulerContext taskSchedulerContext, int schedulerId) {
+      taskSchedulerContexts.add(taskSchedulerContext);
+      uberSchedulerCreated.set(true);
+      testTaskSchedulers.add(yarnTaskScheduler);
+      return uberTaskScheduler;
+    }
+
+    @Override
+    TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext,
+                                            NamedEntityDescriptor taskSchedulerDescriptor, int schedulerId) {
+      taskSchedulerContexts.add(taskSchedulerContext);
+      TaskScheduler taskScheduler = spy(super.createCustomTaskScheduler(taskSchedulerContext, taskSchedulerDescriptor, schedulerId));
+      testTaskSchedulers.add(taskScheduler);
+      return taskScheduler;
+    }
+
+    @Override
+    // Inline handling of events.
+    public void handle(AMSchedulerEvent event) {
+      handleEvent(event);
+    }
+
+    public boolean getUberSchedulerCreated() {
+      return uberSchedulerCreated.get();
+    }
+
+    public boolean getYarnSchedulerCreated() {
+      return yarnSchedulerCreated.get();
+    }
+
+    public int getNumCreateInvocations() {
+      return numCreateInvocations.get();
+    }
+
+    public TaskSchedulerContext getTaskSchedulerContext(int schedulerId) {
+      return taskSchedulerContexts.get(schedulerId);
+    }
+
+    public String getTaskSchedulerName(int schedulerId) {
+      return taskSchedulerNames.get(schedulerId);
+    }
+
+    public TaskScheduler getTestTaskScheduler(int schedulerId) {
+      return testTaskSchedulers.get(schedulerId);
+    }
+  }
+
+  public static class FakeTaskScheduler extends TaskScheduler {
+
+    public FakeTaskScheduler(
+        TaskSchedulerContext taskSchedulerContext) {
+      super(taskSchedulerContext);
+    }
+
+    @Override
+    public Resource getAvailableResources() {
+      return null;
+    }
+
+    @Override
+    public int getClusterNodeCount() {
+      return 0;
+    }
+
+    @Override
+    public void dagComplete() {
+
+    }
+
+    @Override
+    public Resource getTotalResources() {
+      return null;
+    }
+
+    @Override
+    public void blacklistNode(NodeId nodeId) {
+
+    }
+
+    @Override
+    public void unblacklistNode(NodeId nodeId) {
+
+    }
+
+    @Override
+    public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks,
+                             Priority priority, Object containerSignature, Object clientCookie) {
+
+    }
+
+    @Override
+    public void allocateTask(Object task, Resource capability, ContainerId containerId,
+                             Priority priority, Object containerSignature, Object clientCookie) {
+
+    }
+
+    @Override
+    public boolean deallocateTask(Object task, boolean taskSucceeded,
+                                  TaskAttemptEndReason endReason,
+                                  String diagnostics) {
+      return false;
+    }
+
+    @Override
+    public Object deallocateContainer(ContainerId containerId) {
+      return null;
+    }
+
+    @Override
+    public void setShouldUnregister() {
+
+    }
+
+    @Override
+    public boolean hasUnregistered() {
+      return false;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index 13fa4c5..cc88f0d 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -69,14 +69,14 @@ import org.apache.tez.dag.api.TaskCommunicator;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerHeartbeatHandler;
 import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminatedBySystem;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminating;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventNodeFailed;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
 import org.apache.tez.dag.app.rm.AMSchedulerEventType;
-import org.apache.tez.dag.app.rm.NMCommunicatorEventType;
+import org.apache.tez.dag.app.rm.ContainerLauncherEventType;
 import org.apache.tez.dag.history.DAGHistoryEvent;
 import org.apache.tez.dag.history.HistoryEventHandler;
 import org.apache.tez.dag.records.TaskAttemptTerminationCause;
@@ -288,7 +288,7 @@ public class TestAMContainer {
     // Event to NM to stop the container.
     wc.verifyCountAndGetOutgoingEvents(1);
     assertTrue(wc.verifyCountAndGetOutgoingEvents(1).get(0).getType() ==
-        NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+        ContainerLauncherEventType.CONTAINER_STOP_REQUEST);
 
     wc.nmStopSent();
     wc.verifyState(AMContainerState.STOPPING);
@@ -323,7 +323,7 @@ public class TestAMContainer {
     // Event to NM to stop the container.
     wc.verifyCountAndGetOutgoingEvents(1);
     assertTrue(wc.verifyCountAndGetOutgoingEvents(1).get(0).getType() ==
-        NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+        ContainerLauncherEventType.CONTAINER_STOP_REQUEST);
 
     wc.nmStopFailed();
     wc.verifyState(AMContainerState.STOPPING);
@@ -366,7 +366,7 @@ public class TestAMContainer {
     // 1 for NM stop request. 2 TERMINATING to TaskAttempt.
     outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
     verifyUnOrderedOutgoingEventTypes(outgoingEvents,
-        NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
+        ContainerLauncherEventType.CONTAINER_STOP_REQUEST,
         TaskAttemptEventType.TA_CONTAINER_TERMINATING,
         TaskAttemptEventType.TA_CONTAINER_TERMINATING);
     assertTrue(wc.amContainer.isInErrorState());
@@ -405,7 +405,7 @@ public class TestAMContainer {
     // 1 for NM stop request. 2 TERMINATING to TaskAttempt.
     outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
     verifyUnOrderedOutgoingEventTypes(outgoingEvents,
-        NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
+        ContainerLauncherEventType.CONTAINER_STOP_REQUEST,
         TaskAttemptEventType.TA_CONTAINER_TERMINATING,
         TaskAttemptEventType.TA_CONTAINER_TERMINATING);
     assertTrue(wc.amContainer.isInErrorState());
@@ -443,7 +443,7 @@ public class TestAMContainer {
     outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
     verifyUnOrderedOutgoingEventTypes(outgoingEvents,
         TaskAttemptEventType.TA_CONTAINER_TERMINATING,
-        NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+        ContainerLauncherEventType.CONTAINER_STOP_REQUEST);
     // TODO Should this be an RM DE-ALLOCATE instead ?
 
     wc.containerCompleted();
@@ -478,7 +478,7 @@ public class TestAMContainer {
     outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
     verifyUnOrderedOutgoingEventTypes(outgoingEvents,
         TaskAttemptEventType.TA_CONTAINER_TERMINATING,
-        NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+        ContainerLauncherEventType.CONTAINER_STOP_REQUEST);
     // TODO Should this be an RM DE-ALLOCATE instead ?
 
     wc.containerCompleted();
@@ -1194,7 +1194,7 @@ public class TestAMContainer {
     Priority priority;
     Container container;
     ContainerHeartbeatHandler chh;
-    TaskAttemptListener tal;
+    TaskCommunicatorManagerInterface tal;
 
     @SuppressWarnings("rawtypes")
     EventHandler eventHandler;
@@ -1226,7 +1226,7 @@ public class TestAMContainer {
 
       chh = mock(ContainerHeartbeatHandler.class);
 
-      tal = mock(TaskAttemptListener.class);
+      tal = mock(TaskCommunicatorManagerInterface.class);
       TaskCommunicator taskComm = mock(TaskCommunicator.class);
       doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
       doReturn(taskComm).when(tal).getTaskCommunicator(0);
@@ -1440,7 +1440,7 @@ public class TestAMContainer {
     return lr;
   }
 
-  private void verifyUnregisterRunningContainer(TaskAttemptListener tal, ContainerId containerId,
+  private void verifyUnregisterRunningContainer(TaskCommunicatorManagerInterface tal, ContainerId containerId,
                                                 int taskCommId,
                                                 ContainerEndReason containerEndReason,
                                                 String diagContains) {
@@ -1455,7 +1455,7 @@ public class TestAMContainer {
     }
   }
 
-  private void verifyUnregisterTaskAttempt(TaskAttemptListener tal, TezTaskAttemptID taId,
+  private void verifyUnregisterTaskAttempt(TaskCommunicatorManagerInterface tal, TezTaskAttemptID taId,
                                            int taskCommId, TaskAttemptEndReason endReason,
                                            String diagContains) {
     ArgumentCaptor<String> argumentCaptor = ArgumentCaptor.forClass(String.class);

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
index dee4541..0230bb5 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.tez.dag.api.TaskCommunicator;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerHeartbeatHandler;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 
 public class TestAMContainerMap {
 
@@ -42,8 +42,8 @@ public class TestAMContainerMap {
     return mock(ContainerHeartbeatHandler.class);
   }
 
-  private TaskAttemptListener mockTaskAttemptListener() {
-    TaskAttemptListener tal = mock(TaskAttemptListener.class);
+  private TaskCommunicatorManagerInterface mockTaskAttemptListener() {
+    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
     TaskCommunicator taskComm = mock(TaskCommunicator.class);
     doReturn(new InetSocketAddress("localhost", 21000)).when(taskComm).getAddress();
     doReturn(taskComm).when(tal).getTaskCommunicator(0);

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
index def80da..143fcbf 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
@@ -44,7 +44,7 @@ import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.rm.AMSchedulerEventNodeBlacklistUpdate;
 import org.apache.tez.dag.app.rm.AMSchedulerEventType;
-import org.apache.tez.dag.app.rm.TaskSchedulerEventHandler;
+import org.apache.tez.dag.app.rm.TaskSchedulerManager;
 import org.apache.tez.dag.app.rm.container.AMContainerEventNodeFailed;
 import org.apache.tez.dag.app.rm.container.AMContainerEventType;
 import org.apache.tez.dag.app.rm.container.AMContainerMap;
@@ -186,11 +186,11 @@ public class TestAMNodeTracker {
     AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
     doReturn(amNodeTracker).when(appContext).getNodeTracker();
     AMContainerMap amContainerMap = mock(AMContainerMap.class);
-    TaskSchedulerEventHandler taskSchedulerEventHandler =
-        mock(TaskSchedulerEventHandler.class);
+    TaskSchedulerManager taskSchedulerManager =
+        mock(TaskSchedulerManager.class);
     dispatcher.register(AMNodeEventType.class, amNodeTracker);
     dispatcher.register(AMContainerEventType.class, amContainerMap);
-    dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
     amNodeTracker.init(conf);
     amNodeTracker.start();
 
@@ -209,11 +209,11 @@ public class TestAMNodeTracker {
     AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
     doReturn(amNodeTracker).when(appContext).getNodeTracker();
     AMContainerMap amContainerMap = mock(AMContainerMap.class);
-    TaskSchedulerEventHandler taskSchedulerEventHandler =
-        mock(TaskSchedulerEventHandler.class);
+    TaskSchedulerManager taskSchedulerManager =
+        mock(TaskSchedulerManager.class);
     dispatcher.register(AMNodeEventType.class, amNodeTracker);
     dispatcher.register(AMContainerEventType.class, amContainerMap);
-    dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
     amNodeTracker.init(conf);
     amNodeTracker.start();
 
@@ -232,11 +232,11 @@ public class TestAMNodeTracker {
     AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
     doReturn(amNodeTracker).when(appContext).getNodeTracker();
     AMContainerMap amContainerMap = mock(AMContainerMap.class);
-    TaskSchedulerEventHandler taskSchedulerEventHandler =
-        mock(TaskSchedulerEventHandler.class);
+    TaskSchedulerManager taskSchedulerManager =
+        mock(TaskSchedulerManager.class);
     dispatcher.register(AMNodeEventType.class, amNodeTracker);
     dispatcher.register(AMContainerEventType.class, amContainerMap);
-    dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
     amNodeTracker.init(conf);
     amNodeTracker.start();
 
@@ -260,11 +260,11 @@ public class TestAMNodeTracker {
     AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
     doReturn(amNodeTracker).when(appContext).getNodeTracker();
     AMContainerMap amContainerMap = mock(AMContainerMap.class);
-    TaskSchedulerEventHandler taskSchedulerEventHandler =
-        mock(TaskSchedulerEventHandler.class);
+    TaskSchedulerManager taskSchedulerManager =
+        mock(TaskSchedulerManager.class);
     dispatcher.register(AMNodeEventType.class, amNodeTracker);
     dispatcher.register(AMContainerEventType.class, amContainerMap);
-    dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
     amNodeTracker.init(conf);
     amNodeTracker.start();
     try {
@@ -283,11 +283,11 @@ public class TestAMNodeTracker {
     AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
     doReturn(amNodeTracker).when(appContext).getNodeTracker();
     AMContainerMap amContainerMap = mock(AMContainerMap.class);
-    TaskSchedulerEventHandler taskSchedulerEventHandler =
-        mock(TaskSchedulerEventHandler.class);
+    TaskSchedulerManager taskSchedulerManager =
+        mock(TaskSchedulerManager.class);
     dispatcher.register(AMNodeEventType.class, amNodeTracker);
     dispatcher.register(AMContainerEventType.class, amContainerMap);
-    dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
     amNodeTracker.init(conf);
     amNodeTracker.start();
     try {
@@ -306,11 +306,11 @@ public class TestAMNodeTracker {
     AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
     doReturn(amNodeTracker).when(appContext).getNodeTracker();
     AMContainerMap amContainerMap = mock(AMContainerMap.class);
-    TaskSchedulerEventHandler taskSchedulerEventHandler =
-        mock(TaskSchedulerEventHandler.class);
+    TaskSchedulerManager taskSchedulerManager =
+        mock(TaskSchedulerManager.class);
     dispatcher.register(AMNodeEventType.class, amNodeTracker);
     dispatcher.register(AMContainerEventType.class, amContainerMap);
-    dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
     amNodeTracker.init(conf);
     amNodeTracker.start();
     try {

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
index 3b4c768..fb4c08f 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
@@ -60,7 +60,8 @@ import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
 import org.apache.tez.runtime.api.impl.TaskSpec;
 import org.apache.tez.runtime.task.TaskReporter;
-import org.apache.tez.runtime.task.TezTaskRunner;
+import org.apache.tez.runtime.task.TaskRunner2Result;
+import org.apache.tez.runtime.task.TezTaskRunner2;
 import org.apache.tez.service.ContainerRunner;
 import org.apache.tez.dag.api.TezConstants;
 import org.apache.tez.runtime.api.ExecutionContext;
@@ -378,7 +379,7 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
     private final Credentials credentials;
     private final long memoryAvailable;
     private final ListeningExecutorService executor;
-    private volatile TezTaskRunner taskRunner;
+    private volatile TezTaskRunner2 taskRunner;
     private volatile TaskReporter taskReporter;
     private TezTaskUmbilicalProtocol umbilical;
 
@@ -443,7 +444,7 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
           new AtomicLong(0),
           request.getContainerIdString());
 
-      taskRunner = new TezTaskRunner(conf, taskUgi, localDirs,
+      taskRunner = new TezTaskRunner2(conf, taskUgi, localDirs,
           ProtoConverters.getTaskSpecfromProto(request.getTaskSpec()),
           request.getAppAttemptNumber(),
           serviceConsumerMetadata, envMap, startedInputsMap, taskReporter, executor, objectRegistry,
@@ -452,18 +453,20 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
 
       boolean shouldDie;
       try {
-        shouldDie = !taskRunner.run();
+        TaskRunner2Result result = taskRunner.run();
+        LOG.info("TaskRunner2Result: {}", result);
+        shouldDie = result.isContainerShutdownRequested();
         if (shouldDie) {
           LOG.info("Got a shouldDie notification via heartbeats. Shutting down");
           return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null,
               "Asked to die by the AM");
         }
-      } catch (IOException e) {
-        return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE,
-            e, "TaskExecutionFailure: " + e.getMessage());
-      } catch (TezException e) {
-        return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE,
-            e, "TaskExecutionFailure: " + e.getMessage());
+        if (result.getError() != null) {
+          Throwable e = result.getError();
+          return new ContainerExecutionResult(
+              ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE,
+              e, "TaskExecutionFailure: " + e.getMessage());
+        }
       } finally {
         FileSystem.closeAllForUGI(taskUgi);
       }

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
deleted file mode 100644
index aebf6a9..0000000
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
+++ /dev/null
@@ -1,451 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.runtime.task;
-
-import java.io.IOException;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.nio.ByteBuffer;
-import java.security.PrivilegedExceptionAction;
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import com.google.common.base.Throwables;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSError;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.tez.common.CallableWithNdc;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.runtime.LogicalIOProcessorRuntimeTask;
-import org.apache.tez.runtime.RuntimeTask;
-import org.apache.tez.runtime.api.ExecutionContext;
-import org.apache.tez.runtime.api.ObjectRegistry;
-import org.apache.tez.runtime.api.impl.EventMetaData;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.runtime.api.impl.TezEvent;
-import org.apache.tez.runtime.api.impl.TezUmbilical;
-import org.apache.tez.runtime.internals.api.TaskReporterInterface;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Multimap;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-
-public class TezTaskRunner implements TezUmbilical, ErrorReporter {
-
-  private static final Logger LOG = LoggerFactory.getLogger(TezTaskRunner.class);
-
-  private final Configuration tezConf;
-  private final LogicalIOProcessorRuntimeTask task;
-  private final UserGroupInformation ugi;
-
-  private final TaskReporterInterface taskReporter;
-  private final ListeningExecutorService executor;
-  private volatile ListenableFuture<Void> taskFuture;
-  private volatile Thread waitingThread;
-  private volatile Thread taskRunner;
-  private volatile Throwable firstException;
-
-  // Effectively a duplicate check, since hadFatalError does the same thing.
-  private final AtomicBoolean fatalErrorSent = new AtomicBoolean(false);
-  private final AtomicBoolean taskRunning;
-  private final AtomicBoolean shutdownRequested = new AtomicBoolean(false);
-
-  public TezTaskRunner(Configuration tezConf, UserGroupInformation ugi, String[] localDirs,
-      TaskSpec taskSpec, int appAttemptNumber,
-      Map<String, ByteBuffer> serviceConsumerMetadata, Map<String, String> serviceProviderEnvMap,
-      Multimap<String, String> startedInputsMap, TaskReporterInterface taskReporter,
-      ListeningExecutorService executor, ObjectRegistry objectRegistry, String pid,
-      ExecutionContext executionContext, long memAvailable)
-          throws IOException {
-    this.tezConf = tezConf;
-    this.ugi = ugi;
-    this.taskReporter = taskReporter;
-    this.executor = executor;
-    task = new LogicalIOProcessorRuntimeTask(taskSpec, appAttemptNumber, tezConf, localDirs, this,
-        serviceConsumerMetadata, serviceProviderEnvMap, startedInputsMap, objectRegistry, pid,
-        executionContext, memAvailable);
-    taskRunning = new AtomicBoolean(false);
-  }
-
-  /**
-   * @return false if a shutdown message was received during task execution
-   * @throws TezException
-   * @throws IOException
-   */
-  public boolean run() throws InterruptedException, IOException, TezException {
-    waitingThread = Thread.currentThread();
-    taskRunning.set(true);
-    taskReporter.registerTask(task, this);
-    TaskRunnerCallable callable = new TaskRunnerCallable();
-    Throwable failureCause = null;
-    if (!Thread.currentThread().isInterrupted()) {
-      taskFuture = executor.submit(callable);
-    } else {
-      taskReporter.unregisterTask(task.getTaskAttemptID());
-      return isShutdownRequested();
-    }
-    try {
-      taskFuture.get();
-
-      // Task could signal a fatal error and return control, or a failure while registering success.
-      failureCause = firstException;
-
-    } catch (InterruptedException e) {
-      LOG.info("Interrupted while waiting for task to complete. Interrupting task");
-      taskFuture.cancel(true);
-      if (shutdownRequested.get()) {
-        LOG.info("Shutdown requested... returning");
-        return false;
-      }
-      if (firstException != null) {
-        failureCause = firstException;
-      } else {
-        // Interrupted for some other reason.
-        failureCause = e;
-      }
-    } catch (ExecutionException e) {
-      // Exception thrown by the run() method itself.
-      Throwable cause = e.getCause();
-      if (cause instanceof FSError) {
-        // Not immediately fatal, this is an error reported by Hadoop FileSystem
-        failureCause = cause;
-      } else if (cause instanceof Error) {
-        LOG.error("Exception of type Error.", cause);
-        sendFailure(cause, "Fatal Error cause TezChild exit.");
-        throw new TezException("Fatal Error cause TezChild exit.", cause);
-      } else {
-        failureCause = cause;
-      }
-    } finally {
-      // Clear the interrupted status of the blocking thread, in case it is set after the
-      // InterruptedException was invoked.
-      taskReporter.unregisterTask(task.getTaskAttemptID());
-      Thread.interrupted();
-    }
-
-    if (failureCause != null) {
-      if (failureCause instanceof FSError) {
-        // Not immediately fatal, this is an error reported by Hadoop FileSystem
-        LOG.info("Encountered an FSError while executing task: " + task.getTaskAttemptID(),
-            failureCause);
-        throw (FSError) failureCause;
-      } else if (failureCause instanceof Error) {
-        LOG.error("Exception of type Error.", failureCause);
-        sendFailure(failureCause, "Fatal error cause TezChild exit.");
-        throw new TezException("Fatal error cause TezChild exit.", failureCause);
-      } else {
-        if (failureCause instanceof IOException) {
-          throw (IOException) failureCause;
-        } else if (failureCause instanceof TezException) {
-          throw (TezException) failureCause;
-        } else if (failureCause instanceof InterruptedException) {
-          throw (InterruptedException) failureCause;
-        } else {
-          throw new TezException(failureCause);
-        }
-      }
-    }
-    return isShutdownRequested();
-  }
-
-  private boolean isShutdownRequested() {
-    if (shutdownRequested.get()) {
-      LOG.info("Shutdown requested... returning");
-      return false;
-    }
-    return true;
-  }
-
-  private class TaskRunnerCallable extends CallableWithNdc<Void> {
-    @Override
-    protected Void callInternal() throws Exception {
-      try {
-        return ugi.doAs(new PrivilegedExceptionAction<Void>() {
-          @Override
-          public Void run() throws Exception {
-            try {
-              taskRunner = Thread.currentThread();
-              LOG.info("Initializing task" + ", taskAttemptId=" + task.getTaskAttemptID());
-              task.initialize();
-              if (!Thread.currentThread().isInterrupted() && firstException == null) {
-                LOG.info("Running task, taskAttemptId=" + task.getTaskAttemptID());
-                task.run();
-                maybeInterruptWaitingThread();
-
-                LOG.info("Closing task, taskAttemptId=" + task.getTaskAttemptID());
-                task.close();
-                task.setFrameworkCounters();
-              }
-              LOG.info("Task completed, taskAttemptId=" + task.getTaskAttemptID()
-                  + ", fatalErrorOccurred=" + (firstException != null));
-              if (firstException == null) {
-                try {
-                  taskReporter.taskSucceeded(task.getTaskAttemptID());
-                } catch (IOException e) {
-                  LOG.warn("Heartbeat failure caused by communication failure", e);
-                  maybeRegisterFirstException(e);
-                  // Falling off, since the runner thread checks for the registered exception.
-                } catch (TezException e) {
-                  LOG.warn("Heartbeat failure reported by AM", e);
-                  maybeRegisterFirstException(e);
-                  // Falling off, since the runner thread checks for the registered exception.
-                }
-              }
-              return null;
-            } catch (Throwable cause) {
-              if (Thread.currentThread().isInterrupted()) {
-                LOG.info("TaskRunnerCallable interrupted=" + Thread.currentThread().isInterrupted()
-                    + ", shutdownRequest=" + shutdownRequested.get());
-                Thread.currentThread().interrupt();
-                return null;
-              }
-              if (cause instanceof FSError) {
-                // Not immediately fatal, this is an error reported by Hadoop FileSystem
-                maybeRegisterFirstException(cause);
-                LOG.info("Encountered an FSError while executing task: " + task.getTaskAttemptID(),
-                    cause);
-                try {
-                  sendFailure(cause, "FS Error in Child JVM");
-                } catch (Exception ignored) {
-                  // Ignored since another cause is already known
-                  LOG.info(
-                      "Ignoring the following exception since a previous exception is already registered",
-                      ignored.getClass().getName());
-                  if (LOG.isTraceEnabled()) {
-                    LOG.trace("Ignored exception is", ignored);
-                  }
-                }
-                throw (FSError) cause;
-              } else if (cause instanceof Error) {
-                LOG.error("Exception of type Error.", cause);
-                sendFailure(cause, "Fatal Error cause TezChild exit.");
-                throw new TezException("Fatal Error cause TezChild exit.", cause);
-              } else {
-                if (cause instanceof UndeclaredThrowableException) {
-                  cause = ((UndeclaredThrowableException) cause).getCause();
-                }
-                maybeRegisterFirstException(cause);
-                LOG.info("Encountered an error while executing task: " + task.getTaskAttemptID(),
-                    cause);
-                try {
-                  sendFailure(cause, "Failure while running task");
-                } catch (Exception ignored) {
-                  // Ignored since another cause is already known
-                  LOG.info(
-                      "Ignoring the following exception since a previous exception is already registered",
-                      ignored.getClass().getName());
-                  if (LOG.isTraceEnabled()) {
-                    LOG.trace("Ignored exception is", ignored);
-                  }
-                }
-                if (cause instanceof IOException) {
-                  throw (IOException) cause;
-                } else if (cause instanceof TezException) {
-                  throw (TezException) cause;
-                } else {
-                  throw new TezException(cause);
-                }
-              }
-            } finally {
-              task.cleanup();
-            }
-          }
-        });
-      } finally {
-        taskRunning.set(false);
-      }
-    }
-
-    private void maybeInterruptWaitingThread() {
-      /**
-       * Possible that the processor is swallowing InterruptException of taskRunner.interrupt().
-       * In such case, interrupt the waitingThread based on the shutdownRequested flag, so that
-       * entire task gets cancelled.
-       */
-      if (shutdownRequested.get()) {
-        waitingThread.interrupt();
-      }
-    }
-  }
-
-  // should wait until all messages are sent to AM before TezChild shutdown
-  // if this method become async in future
-  private void sendFailure(Throwable t, String message) throws IOException, TezException {
-    if (!fatalErrorSent.getAndSet(true)) {
-      task.setFatalError(t, message);
-      task.setFrameworkCounters();
-      try {
-        taskReporter.taskFailed(task.getTaskAttemptID(), t, message, null);
-      } catch (IOException e) {
-        // A failure reason already exists, Comm error just logged.
-        LOG.warn("Heartbeat failure caused by communication failure", e);
-        throw e;
-      } catch (TezException e) {
-        // A failure reason already exists, Comm error just logged.
-        LOG.warn("Heartbeat failure reported by AM", e);
-        throw e;
-      }
-    } else {
-      LOG.warn("Ignoring fatal error since another error has already been reported", t);
-    }
-  }
-
-  @Override
-  public void addEvents(Collection<TezEvent> events) {
-    if (taskRunning.get()) {
-      taskReporter.addEvents(task.getTaskAttemptID(), events);
-    }
-  }
-
-  @Override
-  public synchronized void signalFatalError(TezTaskAttemptID taskAttemptID, Throwable t,
-      String message, EventMetaData sourceInfo) {
-    // This can be called before a task throws an exception or after it.
-    // If called before a task throws an exception
-    // - ensure a heartbeat is sent with the diagnostics, and sent only once.
-    // - interrupt the waiting thread, and make it throw the reported error.
-    // If called after a task throws an exception, the waiting task has already returned, no point
-    // interrupting it.
-    // This case can be effectively ignored (log), as long as the run() method ends up throwing the
-    // exception.
-    //
-    //
-    if (!fatalErrorSent.getAndSet(true)) {
-      maybeRegisterFirstException(t);
-      try {
-        taskReporter.taskFailed(taskAttemptID, t, getTaskDiagnosticsString(t, message), sourceInfo);
-      } catch (IOException e) {
-        // HeartbeatFailed. Don't need to propagate the heartbeat exception since a task exception
-        // occurred earlier.
-        LOG.warn("Heartbeat failure caused by communication failure", e);
-      } catch (TezException e) {
-        // HeartbeatFailed. Don't need to propagate the heartbeat exception since a task exception
-        // occurred earlier.
-        LOG.warn("Heartbeat failure reported by AM", e);
-      } finally {
-        // Wake up the waiting thread so that it can return control
-        waitingThread.interrupt();
-      }
-    }
-  }
-
-  @Override
-  public boolean canCommit(TezTaskAttemptID taskAttemptID) {
-    if (taskRunning.get()) {
-      try {
-        return taskReporter.canCommit(taskAttemptID);
-      } catch (IOException e) {
-        LOG.warn("Communication failure while trying to commit", e);
-        maybeRegisterFirstException(e);
-        waitingThread.interrupt();
-        // Not informing the task since it will be interrupted.
-        // TODO: Should this be sent to the task as well, current Processors, etc do not handle
-        // interrupts very well.
-        return false;
-      }
-    } else {
-      return false;
-    }
-  }
-
-  @Override
-  public synchronized void reportError(Throwable t) {
-   if (taskRunning.get()) {
-      LOG.error("TaskReporter reported error", t);
-      maybeRegisterFirstException(t);
-      waitingThread.interrupt();
-      // A race is possible between a task succeeding, and a subsequent timed heartbeat failing.
-      // These errors can be ignored, since a task can only succeed if the synchronous taskSucceeded
-      // method does not throw an exception, in which case task success is registered with the AM.
-      // Leave this handling to the next getTask / actual task.
-    } else {
-      LOG.info("Ignoring Communication failure since task with id=" + task.getTaskAttemptID()
-          + " is already complete");
-    }
-  }
-
-  private void abortRunningTask() {
-    if (!taskRunning.get()) {
-      LOG.info("Task is not running");
-      waitingThread.interrupt();
-      return;
-    }
-
-    if (taskRunning.get()) {
-      try {
-        task.abortTask();
-      } catch (Exception e) {
-        LOG.warn("Error when aborting the task", e);
-        try {
-          sendFailure(e, "Error when aborting the task");
-        } catch (Exception ignored) {
-          // Ignored.
-        }
-      }
-    }
-    //Interrupt the relevant threads.  TaskRunner should be interrupted preferably.
-    if (isTaskRunning()) {
-      LOG.info("Interrupting taskRunner=" + taskRunner.getName());
-      taskRunner.interrupt();
-    } else {
-      LOG.info("Interrupting waitingThread=" + waitingThread.getName());
-      waitingThread.interrupt();
-    }
-  }
-
-  private boolean isTaskRunning() {
-    return (taskRunning.get() && task.isRunning());
-  }
-
-  @Override
-  public void shutdownRequested() {
-    shutdownRequested.set(true);
-    abortRunningTask();
-  }
-
-  private String getTaskDiagnosticsString(Throwable t, String message) {
-    String diagnostics;
-    if (t != null && message != null) {
-      diagnostics = "exceptionThrown=" + ExceptionUtils.getStackTrace(t) + ", errorMessage="
-          + message;
-    } else if (t == null && message == null) {
-      diagnostics = "Unknown error";
-    } else {
-      diagnostics = t != null ? "exceptionThrown=" + ExceptionUtils.getStackTrace(t)
-          : " errorMessage=" + message;
-    }
-    return diagnostics;
-  }
-
-  private synchronized void maybeRegisterFirstException(Throwable t) {
-    if (firstException == null) {
-      firstException = t;
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
deleted file mode 100644
index a99416a..0000000
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
+++ /dev/null
@@ -1,362 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.runtime.task;
-
-import static org.apache.tez.runtime.task.TaskExecutionTestHelpers.createTaskReporter;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-import org.apache.tez.dag.records.TezVertexID;
-import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
-import org.apache.tez.runtime.api.impl.InputSpec;
-import org.apache.tez.runtime.api.impl.OutputSpec;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.runtime.common.resources.ScalingAllocator;
-import org.apache.tez.runtime.task.TaskExecutionTestHelpers.TestProcessor;
-import org.apache.tez.runtime.task.TaskExecutionTestHelpers.TezTaskUmbilicalForTest;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-
-// Tests in this class cannot be run in parallel.
-public class TestTaskExecution {
-
-  private static final Logger LOG = LoggerFactory.getLogger(TestTaskExecution.class);
-
-
-
-  private static final Configuration defaultConf = new Configuration();
-  private static final FileSystem localFs;
-  private static final Path workDir;
-
-  private static final ExecutorService taskExecutor = Executors.newFixedThreadPool(1);
-
-  static {
-    defaultConf.set("fs.defaultFS", "file:///");
-    defaultConf.set(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ALLOCATOR_CLASS,
-        ScalingAllocator.class.getName());
-    try {
-      localFs = FileSystem.getLocal(defaultConf);
-      Path wd = new Path(System.getProperty("test.build.data", "/tmp"),
-          TestTaskExecution.class.getSimpleName());
-      workDir = localFs.makeQualified(wd);
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Before
-  public void reset() {
-    TestProcessor.reset();
-  }
-
-  @AfterClass
-  public static void shutdown() {
-    taskExecutor.shutdownNow();
-  }
-
-  @Test(timeout = 5000)
-  public void testSingleSuccessfulTask() throws IOException, InterruptedException, TezException,
-      ExecutionException {
-    ListeningExecutorService executor = null;
-    try {
-      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
-      executor = MoreExecutors.listeningDecorator(rawExecutor);
-      ApplicationId appId = ApplicationId.newInstance(10000, 1);
-      TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
-      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
-      TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
-          TestProcessor.CONF_EMPTY);
-      // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
-      // Signal the processor to go through
-      TestProcessor.signal();
-      boolean result = taskRunnerFuture.get();
-      assertTrue(result);
-      assertNull(taskReporter.currentCallable);
-      umbilical.verifyTaskSuccessEvent();
-    } finally {
-      executor.shutdownNow();
-    }
-  }
-
-  @Test(timeout = 5000)
-  public void testMultipleSuccessfulTasks() throws IOException, InterruptedException, TezException,
-      ExecutionException {
-
-    ListeningExecutorService executor = null;
-    try {
-      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
-      executor = MoreExecutors.listeningDecorator(rawExecutor);
-      ApplicationId appId = ApplicationId.newInstance(10000, 1);
-      TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
-      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
-      TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
-          TestProcessor.CONF_EMPTY);
-      // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
-      // Signal the processor to go through
-      TestProcessor.signal();
-      boolean result = taskRunnerFuture.get();
-      assertTrue(result);
-      assertNull(taskReporter.currentCallable);
-      umbilical.verifyTaskSuccessEvent();
-      umbilical.resetTrackedEvents();
-
-      taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
-          TestProcessor.CONF_EMPTY);
-      // Setup the executor
-      taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
-      // Signal the processor to go through
-      TestProcessor.signal();
-      result = taskRunnerFuture.get();
-      assertTrue(result);
-      assertNull(taskReporter.currentCallable);
-      umbilical.verifyTaskSuccessEvent();
-    } finally {
-      executor.shutdownNow();
-    }
-  }
-
-  // test task failed due to exception in Processor
-  @Test(timeout = 5000)
-  public void testFailedTask() throws IOException, InterruptedException, TezException {
-
-    ListeningExecutorService executor = null;
-    try {
-      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
-      executor = MoreExecutors.listeningDecorator(rawExecutor);
-      ApplicationId appId = ApplicationId.newInstance(10000, 1);
-      TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
-      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
-      TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
-          TestProcessor.CONF_THROW_TEZ_EXCEPTION);
-      // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
-      // Signal the processor to go through
-      TestProcessor.awaitStart();
-      TestProcessor.signal();
-      try {
-        taskRunnerFuture.get();
-        fail("Expecting the task to fail");
-      } catch (ExecutionException e) {
-        Throwable cause = e.getCause();
-        LOG.info(cause.getClass().getName());
-        assertTrue(cause instanceof TezException);
-      }
-
-      assertNull(taskReporter.currentCallable);
-      umbilical.verifyTaskFailedEvent("Failure while running task:org.apache.tez.dag.api.TezException: TezException");
-    } finally {
-      executor.shutdownNow();
-    }
-  }
-
-  // Test task failed due to Processor class not found
-  @Test(timeout = 5000)
-  public void testFailedTask2() throws IOException, InterruptedException, TezException {
-
-    ListeningExecutorService executor = null;
-    try {
-      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
-      executor = MoreExecutors.listeningDecorator(rawExecutor);
-      ApplicationId appId = ApplicationId.newInstance(10000, 1);
-      TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
-      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
-      TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
-          "NotExitedProcessor", TestProcessor.CONF_THROW_TEZ_EXCEPTION);
-      // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
-      try {
-        taskRunnerFuture.get();
-      } catch (ExecutionException e) {
-        Throwable cause = e.getCause();
-        LOG.info(cause.getClass().getName());
-        assertTrue(cause instanceof TezException);
-      }
-      assertNull(taskReporter.currentCallable);
-      umbilical.verifyTaskFailedEvent("Failure while running task:org.apache.tez.dag.api.TezUncheckedException: "
-            + "Unable to load class: NotExitedProcessor");
-    } finally {
-      executor.shutdownNow();
-    }
-  }
-
-  @Test(timeout = 5000)
-  public void testHeartbeatException() throws IOException, InterruptedException, TezException {
-
-    ListeningExecutorService executor = null;
-    try {
-      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
-      executor = MoreExecutors.listeningDecorator(rawExecutor);
-      ApplicationId appId = ApplicationId.newInstance(10000, 1);
-      TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
-      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
-      TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
-          TestProcessor.CONF_EMPTY);
-      // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
-      // Signal the processor to go through
-      TestProcessor.awaitStart();
-      umbilical.signalThrowException();
-      umbilical.awaitRegisteredEvent();
-      // Not signaling an actual start to verify task interruption
-      try {
-        taskRunnerFuture.get();
-        fail("Expecting the task to fail");
-      } catch (ExecutionException e) {
-        Throwable cause = e.getCause();
-        assertTrue(cause instanceof IOException);
-        assertTrue(cause.getMessage().contains(TaskExecutionTestHelpers.HEARTBEAT_EXCEPTION_STRING));
-      }
-      TestProcessor.awaitCompletion();
-      assertTrue(TestProcessor.wasInterrupted());
-      assertNull(taskReporter.currentCallable);
-      // No completion events since umbilical communication already failed.
-      umbilical.verifyNoCompletionEvents();
-    } finally {
-      executor.shutdownNow();
-    }
-  }
-
-  @Test(timeout = 5000)
-  public void testHeartbeatShouldDie() throws IOException, InterruptedException, TezException,
-      ExecutionException {
-
-    ListeningExecutorService executor = null;
-    try {
-      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
-      executor = MoreExecutors.listeningDecorator(rawExecutor);
-      ApplicationId appId = ApplicationId.newInstance(10000, 1);
-      TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
-      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
-      TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
-          TestProcessor.CONF_EMPTY);
-      // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
-      // Signal the processor to go through
-      TestProcessor.awaitStart();
-      umbilical.signalSendShouldDie();
-      umbilical.awaitRegisteredEvent();
-      // Not signaling an actual start to verify task interruption
-
-      boolean result = taskRunnerFuture.get();
-      assertFalse(result);
-
-      TestProcessor.awaitCompletion();
-      assertTrue(TestProcessor.wasInterrupted());
-      assertNull(taskReporter.currentCallable);
-      // TODO Is this statement correct ?
-      // No completion events since shouldDie was requested by the AM, which should have killed the
-      // task.
-      umbilical.verifyNoCompletionEvents();
-    } finally {
-      executor.shutdownNow();
-    }
-  }
-
-  // Potential new tests
-  // Different states - initialization failure, close failure
-  // getTask states
-
-  private static class TaskRunnerCallable1ForTest implements Callable<Boolean> {
-    private final TezTaskRunner taskRunner;
-
-    public TaskRunnerCallable1ForTest(TezTaskRunner taskRunner) {
-      this.taskRunner = taskRunner;
-    }
-
-    @Override
-    public Boolean call() throws Exception {
-      return taskRunner.run();
-    }
-  }
-
-
-
-
-
-  private TezTaskRunner createTaskRunner(ApplicationId appId, TezTaskUmbilicalForTest umbilical,
-      TaskReporter taskReporter, ListeningExecutorService executor, byte[] processorConf)
-      throws IOException {
-    return createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.class.getName(),
-        processorConf);
-  }
-
-  private TezTaskRunner createTaskRunner(ApplicationId appId, TezTaskUmbilicalForTest umbilical,
-      TaskReporter taskReporter, ListeningExecutorService executor, String processorClass, byte[] processorConf) throws IOException{
-    TezConfiguration tezConf = new TezConfiguration(defaultConf);
-    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-    Path testDir = new Path(workDir, UUID.randomUUID().toString());
-    String[] localDirs = new String[] { testDir.toString() };
-
-    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
-    TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
-    TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
-    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
-    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass)
-        .setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
-    TaskSpec taskSpec = new TaskSpec(taskAttemptId, "dagName", "vertexName", -1, processorDescriptor,
-        new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null);
-
-    TezTaskRunner taskRunner = new TezTaskRunner(tezConf, ugi, localDirs, taskSpec, 1,
-        new HashMap<String, ByteBuffer>(), new HashMap<String, String>(), HashMultimap.<String, String> create(), taskReporter,
-        executor, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory());
-    return taskRunner;
-  }
-
-
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
index 12d9d3f..ce9095a 100644
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
@@ -89,7 +89,7 @@ public class TestTaskExecution2 {
     try {
       localFs = FileSystem.getLocal(defaultConf);
       Path wd = new Path(System.getProperty("test.build.data", "/tmp"),
-          TestTaskExecution.class.getSimpleName());
+          TestTaskExecution2.class.getSimpleName());
       workDir = localFs.makeQualified(wd);
     } catch (IOException e) {
       throw new RuntimeException(e);


[2/7] tez git commit: TEZ-2708. Rename classes and variables post TEZ-2003 changes. (sseth)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
index 0a02f9e..0e90681 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
@@ -68,7 +68,7 @@ import org.apache.tez.dag.app.ClusterInfo;
 import org.apache.tez.dag.app.ContainerContext;
 import org.apache.tez.dag.app.ContainerHeartbeatHandler;
 import org.apache.tez.dag.app.DAGAppMasterState;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.dag.TaskAttempt;
 import org.apache.tez.dag.app.rm.YarnTaskSchedulerService.CookieContainerRequest;
 import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.AMRMClientAsyncForTest;
@@ -76,7 +76,7 @@ import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.AMRMClientForTest;
 import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.AlwaysMatchesContainerMatcher;
 import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.CapturingEventHandler;
 import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.TaskSchedulerContextDrainable;
-import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.TaskSchedulerEventHandlerForTest;
+import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.TaskSchedulerManagerForTest;
 import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.TaskSchedulerWithDrainableContext;
 import org.apache.tez.dag.app.rm.container.AMContainerEventAssignTA;
 import org.apache.tez.dag.app.rm.container.AMContainerEventStopRequest;
@@ -136,7 +136,7 @@ public class TestContainerReuse {
     doReturn(conf).when(appContext).getAMConf();
     AMContainerMap amContainerMap = new AMContainerMap(
       mock(ContainerHeartbeatHandler.class),
-      mock(TaskAttemptListener.class),
+      mock(TaskCommunicatorManagerInterface.class),
       new ContainerContextMatcher(), appContext);
     AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
     doReturn(amContainerMap).when(appContext).getAllContainers();
@@ -145,18 +145,18 @@ public class TestContainerReuse {
     doReturn(dagID).when(appContext).getCurrentDAGID();
     doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
 
-    TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
-        new TaskSchedulerEventHandlerForTest(
+    TaskSchedulerManager taskSchedulerManagerReal =
+        new TaskSchedulerManagerForTest(
           appContext, eventHandler, rmClient,
           new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(conf));
-    TaskSchedulerEventHandler taskSchedulerEventHandler =
-        spy(taskSchedulerEventHandlerReal);
-    taskSchedulerEventHandler.init(conf);
-    taskSchedulerEventHandler.start();
+    TaskSchedulerManager taskSchedulerManager =
+        spy(taskSchedulerManagerReal);
+    taskSchedulerManager.init(conf);
+    taskSchedulerManager.start();
 
     TaskSchedulerWithDrainableContext taskScheduler =
       (TaskSchedulerWithDrainableContext)
-        ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+        ((TaskSchedulerManagerForTest) taskSchedulerManager)
         .getSpyTaskScheduler();
     TaskSchedulerContextDrainable drainableAppCallback =
       taskScheduler.getDrainableAppCallback();
@@ -192,10 +192,10 @@ public class TestContainerReuse {
         defaultRack, priority);
 
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrTa11);
+    taskSchedulerManager.handleEvent(lrTa11);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrTa21);
+    taskSchedulerManager.handleEvent(lrTa21);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
 
     Container containerHost1 = createContainer(1, host1[0], resource, priority);
@@ -206,28 +206,28 @@ public class TestContainerReuse {
       Lists.newArrayList(containerHost1, containerHost2));
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler).taskAllocated(
+    verify(taskSchedulerManager).taskAllocated(
       eq(0), eq(ta11), any(Object.class), eq(containerHost1));
-    verify(taskSchedulerEventHandler).taskAllocated(
+    verify(taskSchedulerManager).taskAllocated(
       eq(0), eq(ta21), any(Object.class), eq(containerHost2));
 
     // Adding the event later so that task1 assigned to containerHost1
     // is deterministic.
-    taskSchedulerEventHandler.handleEvent(lrTa31);
+    taskSchedulerManager.handleEvent(lrTa31);
 
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(
             ta11, containerHost1.getId(), TaskAttemptState.SUCCEEDED, null, null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta11, true, null, null);
-    verify(taskSchedulerEventHandler, times(1)).taskAllocated(
+    verify(taskSchedulerManager, times(1)).taskAllocated(
       eq(0), eq(ta31), any(Object.class), eq(containerHost1));
     verify(rmClient, times(0)).releaseAssignedContainer(
       eq(containerHost1.getId()));
     eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
     eventHandler.reset();
 
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
       new AMSchedulerEventTAEnded(ta21, containerHost2.getId(),
         TaskAttemptState.SUCCEEDED, null, null, 0));
 
@@ -235,7 +235,7 @@ public class TestContainerReuse {
     Throwable exception = null;
     while (System.currentTimeMillis() < currentTs + 5000l) {
       try {
-        verify(taskSchedulerEventHandler,
+        verify(taskSchedulerManager,
           times(1)).containerBeingReleased(eq(0), eq(containerHost2.getId()));
         exception = null;
         break;
@@ -245,7 +245,7 @@ public class TestContainerReuse {
     }
     assertTrue("containerHost2 was not released", exception == null);
     taskScheduler.shutdown();
-    taskSchedulerEventHandler.close();
+    taskSchedulerManager.close();
   }
 
   @Test(timeout = 15000l)
@@ -272,7 +272,7 @@ public class TestContainerReuse {
     doReturn(new Configuration(false)).when(appContext).getAMConf();
     AMContainerMap amContainerMap = new AMContainerMap(
       mock(ContainerHeartbeatHandler.class),
-      mock(TaskAttemptListener.class),
+      mock(TaskCommunicatorManagerInterface.class),
       new ContainerContextMatcher(), appContext);
     AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
     doReturn(amContainerMap).when(appContext).getAllContainers();
@@ -281,17 +281,17 @@ public class TestContainerReuse {
     doReturn(dagID).when(appContext).getCurrentDAGID();
     doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
 
-    TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
-      new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient,
+    TaskSchedulerManager taskSchedulerManagerReal =
+      new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient,
         new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(conf));
-    TaskSchedulerEventHandler taskSchedulerEventHandler =
-      spy(taskSchedulerEventHandlerReal);
-    taskSchedulerEventHandler.init(conf);
-    taskSchedulerEventHandler.start();
+    TaskSchedulerManager taskSchedulerManager =
+      spy(taskSchedulerManagerReal);
+    taskSchedulerManager.init(conf);
+    taskSchedulerManager.start();
 
     TaskSchedulerWithDrainableContext taskScheduler =
       (TaskSchedulerWithDrainableContext)
-        ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+        ((TaskSchedulerManagerForTest) taskSchedulerManager)
           .getSpyTaskScheduler();
     TaskSchedulerContextDrainable drainableAppCallback =
       taskScheduler.getDrainableAppCallback();
@@ -321,10 +321,10 @@ public class TestContainerReuse {
       taID31, ta31, resource, host1, defaultRack, priority);
 
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrTa11);
+    taskSchedulerManager.handleEvent(lrTa11);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrTa21);
+    taskSchedulerManager.handleEvent(lrTa21);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
 
     Container containerHost1 = createContainer(1, host1[0], resource, priority);
@@ -334,26 +334,26 @@ public class TestContainerReuse {
     taskScheduler.onContainersAllocated(Lists.newArrayList(containerHost1, containerHost2));
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta11), any(Object.class), eq(containerHost1));
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta21), any(Object.class),
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta11), any(Object.class), eq(containerHost1));
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta21), any(Object.class),
         eq(containerHost2));
 
     // Adding the event later so that task1 assigned to containerHost1 is deterministic.
-    taskSchedulerEventHandler.handleEvent(lrTa31);
+    taskSchedulerManager.handleEvent(lrTa31);
 
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta21, containerHost2.getId(),
             TaskAttemptState.SUCCEEDED, null, null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta21, true, null, null);
-    verify(taskSchedulerEventHandler, times(0)).taskAllocated(
+    verify(taskSchedulerManager, times(0)).taskAllocated(
         eq(0), eq(ta31), any(Object.class), eq(containerHost2));
     verify(rmClient, times(1)).releaseAssignedContainer(
       eq(containerHost2.getId()));
     eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
 
     taskScheduler.shutdown();
-    taskSchedulerEventHandler.close();
+    taskSchedulerManager.close();
   }
 
   @Test(timeout = 10000l)
@@ -375,7 +375,7 @@ public class TestContainerReuse {
     AppContext appContext = mock(AppContext.class);
     doReturn(new Configuration(false)).when(appContext).getAMConf();
     AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class),
-        mock(TaskAttemptListener.class), new ContainerContextMatcher(), appContext);
+        mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
     AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
     doReturn(amContainerMap).when(appContext).getAllContainers();
     doReturn(amNodeTracker).when(appContext).getNodeTracker();
@@ -383,12 +383,13 @@ public class TestContainerReuse {
     doReturn(dagID).when(appContext).getCurrentDAGID();
     doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
 
-    TaskSchedulerEventHandler taskSchedulerEventHandlerReal = new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
-    TaskSchedulerEventHandler taskSchedulerEventHandler = spy(taskSchedulerEventHandlerReal);
-    taskSchedulerEventHandler.init(tezConf);
-    taskSchedulerEventHandler.start();
+    TaskSchedulerManager
+        taskSchedulerManagerReal = new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
+    TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal);
+    taskSchedulerManager.init(tezConf);
+    taskSchedulerManager.start();
 
-    TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+    TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerManagerForTest) taskSchedulerManager)
         .getSpyTaskScheduler();
     TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
     AtomicBoolean drainNotifier = new AtomicBoolean(false);
@@ -423,10 +424,10 @@ public class TestContainerReuse {
     TaskAttempt ta14 = mock(TaskAttempt.class);
     AMSchedulerEventTALaunchRequest lrEvent4 = createLaunchRequestEvent(taID14, ta14, resource1, host2, racks, priority1);
 
-    taskSchedulerEventHandler.handleEvent(lrEvent1);
-    taskSchedulerEventHandler.handleEvent(lrEvent2);
-    taskSchedulerEventHandler.handleEvent(lrEvent3);
-    taskSchedulerEventHandler.handleEvent(lrEvent4);
+    taskSchedulerManager.handleEvent(lrEvent1);
+    taskSchedulerManager.handleEvent(lrEvent2);
+    taskSchedulerManager.handleEvent(lrEvent3);
+    taskSchedulerManager.handleEvent(lrEvent4);
 
     Container container1 = createContainer(1, "host1", resource1, priority1);
 
@@ -435,39 +436,39 @@ public class TestContainerReuse {
     taskScheduler.onContainersAllocated(Collections.singletonList(container1));
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta11), any(Object.class),
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta11), any(Object.class),
         eq(container1));
 
     // Task assigned to container completed successfully. Container should be re-used.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED, null,
             null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta11, true, null, null);
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta12), any(Object.class), eq(container1));
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta12), any(Object.class), eq(container1));
     verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
     eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
     eventHandler.reset();
 
     // Task assigned to container completed successfully.
     // Verify reuse across hosts.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta12, container1.getId(), TaskAttemptState.SUCCEEDED, null,
             null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta12, true, null, null);
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta13), any(Object.class),
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta13), any(Object.class),
         eq(container1));
     verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
     eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
     eventHandler.reset();
 
     // Verify no re-use if a previous task fails.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta13, container1.getId(), TaskAttemptState.FAILED, null,
             "TIMEOUT", 0));
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(0), eq(ta14), any(Object.class),
+    verify(taskSchedulerManager, times(0)).taskAllocated(eq(0), eq(ta14), any(Object.class),
         eq(container1));
     verifyDeAllocateTask(taskScheduler, ta13, false, null, "TIMEOUT");
     verify(rmClient).releaseAssignedContainer(eq(container1.getId()));
@@ -481,11 +482,11 @@ public class TestContainerReuse {
     taskScheduler.onContainersAllocated(Collections.singletonList(container2));
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta14), any(Object.class),
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta14), any(Object.class),
         eq(container2));
 
     // Task assigned to container completed successfully. No pending requests. Container should be released.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta14, container2.getId(), TaskAttemptState.SUCCEEDED, null,
             null, 0));
     drainableAppCallback.drain();
@@ -495,7 +496,7 @@ public class TestContainerReuse {
     eventHandler.reset();
 
     taskScheduler.shutdown();
-    taskSchedulerEventHandler.close();
+    taskSchedulerManager.close();
   }
 
   @Test(timeout = 10000l)
@@ -521,7 +522,7 @@ public class TestContainerReuse {
     AppContext appContext = mock(AppContext.class);
     doReturn(new Configuration(false)).when(appContext).getAMConf();
     AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class),
-        mock(TaskAttemptListener.class), new ContainerContextMatcher(), appContext);
+        mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
     AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
     doReturn(amContainerMap).when(appContext).getAllContainers();
     doReturn(amNodeTracker).when(appContext).getNodeTracker();
@@ -530,14 +531,14 @@ public class TestContainerReuse {
     doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
 
     //Use ContainerContextMatcher here.  Otherwise it would not match the JVM options
-    TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
-        new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient, new ContainerContextMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
-    TaskSchedulerEventHandler taskSchedulerEventHandler = spy(taskSchedulerEventHandlerReal);
-    taskSchedulerEventHandler.init(tezConf);
-    taskSchedulerEventHandler.start();
+    TaskSchedulerManager taskSchedulerManagerReal =
+        new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, new ContainerContextMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
+    TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal);
+    taskSchedulerManager.init(tezConf);
+    taskSchedulerManager.start();
 
     TaskSchedulerWithDrainableContext taskScheduler =
-        (TaskSchedulerWithDrainableContext) ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+        (TaskSchedulerWithDrainableContext) ((TaskSchedulerManagerForTest) taskSchedulerManager)
           .getSpyTaskScheduler();
     TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
     AtomicBoolean drainNotifier = new AtomicBoolean(false);
@@ -573,10 +574,10 @@ public class TestContainerReuse {
         createLaunchRequestEvent(taID12, ta12, resource1, host1, racks, priority1);
 
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrEvent1);
+    taskSchedulerManager.handleEvent(lrEvent1);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrEvent2);
+    taskSchedulerManager.handleEvent(lrEvent2);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
 
     Container container1 = createContainer(1, "host1", resource1, priority1);
@@ -586,16 +587,16 @@ public class TestContainerReuse {
     taskScheduler.onContainersAllocated(Collections.singletonList(container1));
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta11), any(Object.class),
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta11), any(Object.class),
         eq(container1));
 
     // First task had profiling on. This container can not be reused further.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED, null,
             null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta11, true, null, null);
-    verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(0), eq(ta12), any(Object.class),
+    verify(taskSchedulerManager, times(0)).taskAllocated(eq(0), eq(ta12), any(Object.class),
         eq(container1));
     verify(rmClient, times(1)).releaseAssignedContainer(eq(container1.getId()));
     eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
@@ -623,10 +624,10 @@ public class TestContainerReuse {
 
     Container container2 = createContainer(2, "host2", resource1, priority1);
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrEvent3);
+    taskSchedulerManager.handleEvent(lrEvent3);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrEvent4);
+    taskSchedulerManager.handleEvent(lrEvent4);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
 
     // Container started
@@ -634,16 +635,16 @@ public class TestContainerReuse {
     taskScheduler.onContainersAllocated(Collections.singletonList(container2));
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta13), any(Object.class), eq(container2));
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta13), any(Object.class), eq(container2));
 
     // Verify that the container can not be reused when profiling option is turned on
     // Even for 2 tasks having same profiling option can have container reusability.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta13, container2.getId(), TaskAttemptState.SUCCEEDED, null,
             null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta13, true, null, null);
-    verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(0), eq(ta14), any(Object.class),
+    verify(taskSchedulerManager, times(0)).taskAllocated(eq(0), eq(ta14), any(Object.class),
         eq(container2));
     verify(rmClient, times(1)).releaseAssignedContainer(eq(container2.getId()));
     eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
@@ -675,27 +676,27 @@ public class TestContainerReuse {
 
     // Container started
     Container container3 = createContainer(2, "host3", resource1, priority1);
-    taskSchedulerEventHandler.handleEvent(lrEvent5);
-    taskSchedulerEventHandler.handleEvent(lrEvent6);
+    taskSchedulerManager.handleEvent(lrEvent5);
+    taskSchedulerManager.handleEvent(lrEvent6);
 
     drainNotifier.set(false);
     taskScheduler.onContainersAllocated(Collections.singletonList(container3));
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta15), any(Object.class),
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta15), any(Object.class),
         eq(container3));
 
     //Ensure task 6 (of vertex 1) is allocated to same container
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta15, container3.getId(), TaskAttemptState.SUCCEEDED, null,
             null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta15, true, null, null);
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta16), any(Object.class), eq(container3));
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta16), any(Object.class), eq(container3));
     eventHandler.reset();
 
     taskScheduler.shutdown();
-    taskSchedulerEventHandler.close();
+    taskSchedulerManager.close();
   }
 
   @Test(timeout = 30000l)
@@ -721,7 +722,7 @@ public class TestContainerReuse {
     doReturn(new Configuration(false)).when(appContext).getAMConf();
     AMContainerMap amContainerMap = new AMContainerMap(
         mock(ContainerHeartbeatHandler.class),
-        mock(TaskAttemptListener.class),
+        mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appContext);
     AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
     doReturn(amContainerMap).when(appContext).getAllContainers();
@@ -730,18 +731,18 @@ public class TestContainerReuse {
     doReturn(dagID).when(appContext).getCurrentDAGID();
     doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
 
-    TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
-        new TaskSchedulerEventHandlerForTest(
+    TaskSchedulerManager taskSchedulerManagerReal =
+        new TaskSchedulerManagerForTest(
           appContext, eventHandler, rmClient,
           new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
-    TaskSchedulerEventHandler taskSchedulerEventHandler =
-        spy(taskSchedulerEventHandlerReal);
-    taskSchedulerEventHandler.init(tezConf);
-    taskSchedulerEventHandler.start();
+    TaskSchedulerManager taskSchedulerManager =
+        spy(taskSchedulerManagerReal);
+    taskSchedulerManager.init(tezConf);
+    taskSchedulerManager.start();
 
     TaskSchedulerWithDrainableContext taskScheduler =
       (TaskSchedulerWithDrainableContext)
-        ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+        ((TaskSchedulerManagerForTest) taskSchedulerManager)
         .getSpyTaskScheduler();
     TaskSchedulerContextDrainable drainableAppCallback =
       taskScheduler.getDrainableAppCallback();
@@ -772,7 +773,7 @@ public class TestContainerReuse {
 
     // Send launch request for task 1 only, deterministic assignment to this task.
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrEvent11);
+    taskSchedulerManager.handleEvent(lrEvent11);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
 
     Container container1 = createContainer(1, "randomHost", resource1, priority);
@@ -782,21 +783,21 @@ public class TestContainerReuse {
     taskScheduler.onContainersAllocated(Collections.singletonList(container1));
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler).taskAllocated(
+    verify(taskSchedulerManager).taskAllocated(
         eq(0), eq(ta11), any(Object.class), eq(container1));
 
     // Send launch request for task2 (vertex2)
-    taskSchedulerEventHandler.handleEvent(lrEvent12);
+    taskSchedulerManager.handleEvent(lrEvent12);
 
     // Task assigned to container completed successfully.
     // Container should not be immediately assigned to task 2
     // until delay expires.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta11, container1.getId(),
             TaskAttemptState.SUCCEEDED, null, null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta11, true, null, null);
-    verify(taskSchedulerEventHandler, times(0)).taskAllocated(
+    verify(taskSchedulerManager, times(0)).taskAllocated(
         eq(0), eq(ta12), any(Object.class), eq(container1));
     verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
     eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
@@ -804,11 +805,11 @@ public class TestContainerReuse {
 
     LOG.info("Sleeping to ensure that the scheduling loop runs");
     Thread.sleep(3000l);
-    verify(taskSchedulerEventHandler).taskAllocated(
+    verify(taskSchedulerManager).taskAllocated(
         eq(0), eq(ta12), any(Object.class), eq(container1));
 
     // TA12 completed.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
       new AMSchedulerEventTAEnded(ta12, container1.getId(),
         TaskAttemptState.SUCCEEDED, null, null, 0));
     drainableAppCallback.drain();
@@ -818,7 +819,7 @@ public class TestContainerReuse {
     eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
 
     taskScheduler.shutdown();
-    taskSchedulerEventHandler.close();
+    taskSchedulerManager.close();
   }
 
   @Test(timeout = 30000l)
@@ -845,7 +846,7 @@ public class TestContainerReuse {
     doReturn(new Configuration(false)).when(appContext).getAMConf();
     AMContainerMap amContainerMap = new AMContainerMap(
       mock(ContainerHeartbeatHandler.class),
-      mock(TaskAttemptListener.class),
+      mock(TaskCommunicatorManagerInterface.class),
       new ContainerContextMatcher(), appContext);
     AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
     doReturn(amContainerMap).when(appContext).getAllContainers();
@@ -855,17 +856,17 @@ public class TestContainerReuse {
     doReturn(dagID).when(appContext).getCurrentDAGID();
     doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
 
-    TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
-      new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient,
+    TaskSchedulerManager taskSchedulerManagerReal =
+      new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient,
         new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
-    TaskSchedulerEventHandler taskSchedulerEventHandler =
-      spy(taskSchedulerEventHandlerReal);
-    taskSchedulerEventHandler.init(tezConf);
-    taskSchedulerEventHandler.start();
+    TaskSchedulerManager taskSchedulerManager =
+      spy(taskSchedulerManagerReal);
+    taskSchedulerManager.init(tezConf);
+    taskSchedulerManager.start();
 
     TaskSchedulerWithDrainableContext taskScheduler =
       (TaskSchedulerWithDrainableContext)
-        ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+        ((TaskSchedulerManagerForTest) taskSchedulerManager)
           .getSpyTaskScheduler();
     TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
 
@@ -900,7 +901,7 @@ public class TestContainerReuse {
 
     // Send launch request for task 1 onle, deterministic assignment to this task.
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrEvent11);
+    taskSchedulerManager.handleEvent(lrEvent11);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
 
     Container container1 = createContainer(1, host1[0], resource1, priority1);
@@ -910,25 +911,25 @@ public class TestContainerReuse {
     taskScheduler.onContainersAllocated(Collections.singletonList(container1));
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler).taskAllocated(
+    verify(taskSchedulerManager).taskAllocated(
         eq(0), eq(ta11), any(Object.class), eq(container1));
 
     // Send launch request for task2 (vertex2)
-    taskSchedulerEventHandler.handleEvent(lrEvent21);
+    taskSchedulerManager.handleEvent(lrEvent21);
 
     // Task assigned to container completed successfully.
     // Container should  be assigned to task21.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta11, container1.getId(),
             TaskAttemptState.SUCCEEDED, null, null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta11, true, null, null);
-    verify(taskSchedulerEventHandler).taskAllocated(
+    verify(taskSchedulerManager).taskAllocated(
         eq(0), eq(ta21), any(Object.class), eq(container1));
     verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
 
     // Task 2 completes.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta21, container1.getId(),
             TaskAttemptState.SUCCEEDED, null, null, 0));
     verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
@@ -939,7 +940,7 @@ public class TestContainerReuse {
     verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
 
     taskScheduler.shutdown();
-    taskSchedulerEventHandler.close();
+    taskSchedulerManager.close();
   }
   
   @Test(timeout = 30000l)
@@ -962,7 +963,7 @@ public class TestContainerReuse {
     doReturn(new Configuration(false)).when(appContext).getAMConf();
     ChangingDAGIDAnswer dagIDAnswer = new ChangingDAGIDAnswer(dagID1);
     AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class),
-        mock(TaskAttemptListener.class), new ContainerContextMatcher(), appContext);
+        mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
     AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
     doReturn(amContainerMap).when(appContext).getAllContainers();
     doReturn(amNodeTracker).when(appContext).getNodeTracker();
@@ -971,12 +972,13 @@ public class TestContainerReuse {
     doAnswer(dagIDAnswer).when(appContext).getCurrentDAGID();
     doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
     
-    TaskSchedulerEventHandler taskSchedulerEventHandlerReal = new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
-    TaskSchedulerEventHandler taskSchedulerEventHandler = spy(taskSchedulerEventHandlerReal);
-    taskSchedulerEventHandler.init(tezConf);
-    taskSchedulerEventHandler.start();
+    TaskSchedulerManager
+        taskSchedulerManagerReal = new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
+    TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal);
+    taskSchedulerManager.init(tezConf);
+    taskSchedulerManager.start();
 
-    TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+    TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerManagerForTest) taskSchedulerManager)
         .getSpyTaskScheduler();
     TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
     AtomicBoolean drainNotifier = new AtomicBoolean(false);
@@ -1013,10 +1015,10 @@ public class TestContainerReuse {
     AMSchedulerEventTALaunchRequest lrEvent12 = createLaunchRequestEvent(taID112, ta112, resource1, host1, racks, priority1, dag1LRs);
 
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrEvent11);
+    taskSchedulerManager.handleEvent(lrEvent11);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrEvent12);
+    taskSchedulerManager.handleEvent(lrEvent12);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
 
     Container container1 = createContainer(1, "host1", resource1, priority1);
@@ -1026,17 +1028,17 @@ public class TestContainerReuse {
     taskScheduler.onContainersAllocated(Collections.singletonList(container1));
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta111), any(Object.class), eq(container1));
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta111), any(Object.class), eq(container1));
     assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
     assertEquals(1, assignEvent.getRemoteTaskLocalResources().size());
     
     // Task assigned to container completed successfully. Container should be re-used.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED, null,
             null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta111, true, null, null);
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta112), any(Object.class), eq(container1));
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta112), any(Object.class), eq(container1));
     verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
     eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
     assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
@@ -1045,7 +1047,7 @@ public class TestContainerReuse {
 
     // Task assigned to container completed successfully.
     // Verify reuse across hosts.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED, null,
             null, 0));
     drainableAppCallback.drain();
@@ -1074,14 +1076,14 @@ public class TestContainerReuse {
     TaskAttempt ta212 = mock(TaskAttempt.class);
     AMSchedulerEventTALaunchRequest lrEvent22 = createLaunchRequestEvent(taID212, ta212, resource1, host1, racks, priority1, dag2LRs);
 
-    taskSchedulerEventHandler.handleEvent(lrEvent21);
-    taskSchedulerEventHandler.handleEvent(lrEvent22);
+    taskSchedulerManager.handleEvent(lrEvent21);
+    taskSchedulerManager.handleEvent(lrEvent22);
     drainableAppCallback.drain();
 
     // TODO This is terrible, need a better way to ensure the scheduling loop has run
     LOG.info("Sleeping to ensure that the scheduling loop runs");
     Thread.sleep(6000l);
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta211), any(Object.class),
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta211), any(Object.class),
         eq(container1));
     verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
     eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
@@ -1089,12 +1091,12 @@ public class TestContainerReuse {
     assertEquals(2, assignEvent.getRemoteTaskLocalResources().size());
     eventHandler.reset();
 
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta211, container1.getId(), TaskAttemptState.SUCCEEDED, null,
             null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta211, true, null, null);
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta212), any(Object.class), eq(container1));
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta212), any(Object.class), eq(container1));
     verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
     eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
     assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
@@ -1102,7 +1104,7 @@ public class TestContainerReuse {
     eventHandler.reset();
 
     taskScheduler.shutdown();
-    taskSchedulerEventHandler.close();
+    taskSchedulerManager.close();
   }
 
   @Test(timeout = 30000l)
@@ -1125,7 +1127,7 @@ public class TestContainerReuse {
     doReturn(new Configuration(false)).when(appContext).getAMConf();
     ChangingDAGIDAnswer dagIDAnswer = new ChangingDAGIDAnswer(dagID1);
     AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class),
-        mock(TaskAttemptListener.class), new ContainerContextMatcher(), appContext);
+        mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
     AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
     doReturn(amContainerMap).when(appContext).getAllContainers();
     doReturn(amNodeTracker).when(appContext).getNodeTracker();
@@ -1134,14 +1136,14 @@ public class TestContainerReuse {
     doAnswer(dagIDAnswer).when(appContext).getCurrentDAGID();
     doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
 
-    TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
-        new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient,
+    TaskSchedulerManager taskSchedulerManagerReal =
+        new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient,
             new ContainerContextMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
-    TaskSchedulerEventHandler taskSchedulerEventHandler = spy(taskSchedulerEventHandlerReal);
-    taskSchedulerEventHandler.init(tezConf);
-    taskSchedulerEventHandler.start();
+    TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal);
+    taskSchedulerManager.init(tezConf);
+    taskSchedulerManager.start();
 
-    TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+    TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerManagerForTest) taskSchedulerManager)
         .getSpyTaskScheduler();
     TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
     AtomicBoolean drainNotifier = new AtomicBoolean(false);
@@ -1185,10 +1187,10 @@ public class TestContainerReuse {
     AMSchedulerEventTALaunchRequest lrEvent12 = createLaunchRequestEvent(taID112, ta112, resource1, host1, racks, priority1, v12LR);
 
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrEvent11);
+    taskSchedulerManager.handleEvent(lrEvent11);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainNotifier.set(false);
-    taskSchedulerEventHandler.handleEvent(lrEvent12);
+    taskSchedulerManager.handleEvent(lrEvent12);
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
 
     Container container1 = createContainer(1, "host1", resource1, priority1);
@@ -1199,17 +1201,17 @@ public class TestContainerReuse {
     taskScheduler.onContainersAllocated(Collections.singletonList(container1));
     TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta111), any(Object.class), eq(container1));
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta111), any(Object.class), eq(container1));
     assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
     assertEquals(1, assignEvent.getRemoteTaskLocalResources().size());
 
     // Task assigned to container completed successfully. Container should be re-used.
-    taskSchedulerEventHandler.handleEvent(
+    taskSchedulerManager.handleEvent(
         new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED, null,
             null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta111, true, null, null);
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta112), any(Object.class), eq(container1));
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta112), any(Object.class), eq(container1));
     verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
     eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
     assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
@@ -1218,7 +1220,7 @@ public class TestContainerReuse {
 
     // Task assigned to container completed successfully.
     // Verify reuse across hosts.
-    taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED, null, null, 0));
+    taskSchedulerManager.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED, null, null, 0));
     drainableAppCallback.drain();
     verifyDeAllocateTask(taskScheduler, ta112, true, null, null);
     verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
@@ -1243,7 +1245,7 @@ public class TestContainerReuse {
     AMSchedulerEventTALaunchRequest lrEvent21 = createLaunchRequestEvent(taID211, ta211, resource1,
         host1, racks, priority1, v21LR);
 
-    taskSchedulerEventHandler.handleEvent(lrEvent21);
+    taskSchedulerManager.handleEvent(lrEvent21);
     drainableAppCallback.drain();
 
     // TODO This is terrible, need a better way to ensure the scheduling loop has run
@@ -1254,11 +1256,11 @@ public class TestContainerReuse {
 
     Thread.sleep(6000l);
     verify(rmClient, times(1)).releaseAssignedContainer(eq(container1.getId()));
-    verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta211), any(Object.class), eq(container2));
+    verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta211), any(Object.class), eq(container2));
     eventHandler.reset();
 
     taskScheduler.shutdown();
-    taskSchedulerEventHandler.close();
+    taskSchedulerManager.close();
   }
 
   @Test(timeout = 10000l)
@@ -1281,7 +1283,7 @@ public class TestContainerReuse {
     AppContext appContext = mock(AppContext.class);
     doReturn(new Configuration(false)).when(appContext).getAMConf();
     AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class),
-        mock(TaskAttemptListener.class), new ContainerContextMatcher(), appContext);
+        mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
     AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
     doReturn(amContainerMap).when(appContext).getAllContainers();
     doReturn(amNodeTracker).when(appContext).getNodeTracker();
@@ -1290,15 +1292,15 @@ public class TestContainerReuse {
     doReturn(dagID).when(appContext).getCurrentDAGID();
     doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
 
-    TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
-        new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient,
+    TaskSchedulerManager taskSchedulerManagerReal =
+        new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient,
             new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
-    TaskSchedulerEventHandler taskSchedulerEventHandler = spy(taskSchedulerEventHandlerReal);
-    taskSchedulerEventHandler.init(tezConf);
-    taskSchedulerEventHandler.start();
+    TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal);
+    taskSchedulerManager.init(tezConf);
+    taskSchedulerManager.start();
 
     TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext)
-        ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+        ((TaskSchedulerManagerForTest) taskSchedulerManager)
         .getSpyTaskScheduler();
     TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
     AtomicBoolean drainNotifier = new AtomicBoolean(false);
@@ -1318,7 +1320,7 @@ public class TestContainerReuse {
     TaskAttempt ta11 = mock(TaskAttempt.class);
     AMSchedulerEventTALaunchRequest lrEvent1 = createLaunchRequestEvent(taID11, ta11, resource1,
         host1, racks, priority1);
-    taskSchedulerEventHandler.handleEvent(lrEvent1);
+    taskSchedulerManager.handleEvent(lrEvent1);
 
     Container container1 = createContainer(1, "host1", resource1, priority1);
 
@@ -1326,10 +1328,10 @@ public class TestContainerReuse {
     drainNotifier.set(false);
     taskScheduler.onContainersAllocated(Collections.singletonList(container1));
     drainableAppCallback.drain();
-    verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(0), eq(ta11),
+    verify(taskSchedulerManager, times(0)).taskAllocated(eq(0), eq(ta11),
         any(Object.class), eq(container1));
     taskScheduler.shutdown();
-    taskSchedulerEventHandler.close();
+    taskSchedulerManager.close();
   }
 
   private Container createContainer(int id, String host, Resource resource, Priority priority) {

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
deleted file mode 100644
index c85be6c..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
+++ /dev/null
@@ -1,707 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.common.ContainerSignatureMatcher;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TaskLocationHint;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.client.DAGClientServer;
-import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
-import org.apache.tez.dag.app.dag.TaskAttempt;
-import org.apache.tez.dag.app.dag.impl.TaskAttemptImpl;
-import org.apache.tez.dag.app.dag.impl.TaskImpl;
-import org.apache.tez.dag.app.dag.impl.VertexImpl;
-import org.apache.tez.dag.app.rm.container.AMContainer;
-import org.apache.tez.dag.app.rm.container.AMContainerEventAssignTA;
-import org.apache.tez.dag.app.rm.container.AMContainerEventCompleted;
-import org.apache.tez.dag.app.rm.container.AMContainerEventType;
-import org.apache.tez.dag.app.rm.container.AMContainerMap;
-import org.apache.tez.dag.app.rm.container.AMContainerState;
-import org.apache.tez.dag.app.web.WebUIService;
-import org.apache.tez.dag.records.TaskAttemptTerminationCause;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-import org.apache.tez.dag.records.TezVertexID;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.serviceplugins.api.TaskScheduler;
-import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-@SuppressWarnings("rawtypes")
-public class TestTaskSchedulerEventHandler {
-  
-  class TestEventHandler implements EventHandler{
-    List<Event> events = Lists.newLinkedList();
-    @Override
-    public void handle(Event event) {
-      events.add(event);
-    }
-  }
-  
-  class MockTaskSchedulerEventHandler extends TaskSchedulerEventHandler {
-
-    final AtomicBoolean notify = new AtomicBoolean(false);
-    
-    public MockTaskSchedulerEventHandler(AppContext appContext,
-        DAGClientServer clientService, EventHandler eventHandler,
-        ContainerSignatureMatcher containerSignatureMatcher, WebUIService webUI) {
-      super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI,
-          Lists.newArrayList(new NamedEntityDescriptor("FakeDescriptor", null)), false);
-    }
-
-    @Override
-    protected void instantiateSchedulers(String host, int port, String trackingUrl,
-                                         AppContext appContext) {
-      taskSchedulers[0] = mockTaskScheduler;
-      taskSchedulerServiceWrappers[0] = new ServicePluginLifecycleAbstractService<>(taskSchedulers[0]);
-    }
-    
-    @Override
-    protected void notifyForTest() {
-      synchronized (notify) {
-        notify.set(true);
-        notify.notifyAll();
-      }
-    }
-    
-  }
-
-  AppContext mockAppContext;
-  DAGClientServer mockClientService;
-  TestEventHandler mockEventHandler;
-  ContainerSignatureMatcher mockSigMatcher;
-  MockTaskSchedulerEventHandler schedulerHandler;
-  TaskScheduler mockTaskScheduler;
-  AMContainerMap mockAMContainerMap;
-  WebUIService mockWebUIService;
-
-  @Before
-  public void setup() {
-    mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
-    doReturn(new Configuration(false)).when(mockAppContext).getAMConf();
-    mockClientService = mock(DAGClientServer.class);
-    mockEventHandler = new TestEventHandler();
-    mockSigMatcher = mock(ContainerSignatureMatcher.class);
-    mockTaskScheduler = mock(TaskScheduler.class);
-    mockAMContainerMap = mock(AMContainerMap.class);
-    mockWebUIService = mock(WebUIService.class);
-    when(mockAppContext.getAllContainers()).thenReturn(mockAMContainerMap);
-    when(mockClientService.getBindAddress()).thenReturn(new InetSocketAddress(10000));
-    schedulerHandler = new MockTaskSchedulerEventHandler(
-        mockAppContext, mockClientService, mockEventHandler, mockSigMatcher, mockWebUIService);
-  }
-
-  @Test(timeout = 5000)
-  public void testSimpleAllocate() throws Exception {
-    Configuration conf = new Configuration(false);
-    schedulerHandler.init(conf);
-    schedulerHandler.start();
-
-    TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
-    TezTaskAttemptID mockAttemptId = mock(TezTaskAttemptID.class);
-    when(mockAttemptId.getId()).thenReturn(0);
-    when(mockTaskAttempt.getID()).thenReturn(mockAttemptId);
-    Resource resource = Resource.newInstance(1024, 1);
-    ContainerContext containerContext =
-        new ContainerContext(new HashMap<String, LocalResource>(), new Credentials(),
-            new HashMap<String, String>(), "");
-    int priority = 10;
-    TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(), null);
-
-    ContainerId mockCId = mock(ContainerId.class);
-    Container container = mock(Container.class);
-    when(container.getId()).thenReturn(mockCId);
-
-    AMContainer mockAMContainer = mock(AMContainer.class);
-    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
-    when(mockAMContainer.getState()).thenReturn(AMContainerState.IDLE);
-
-    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
-
-    AMSchedulerEventTALaunchRequest lr =
-        new AMSchedulerEventTALaunchRequest(mockAttemptId, resource, null, mockTaskAttempt, locHint,
-            priority, containerContext, 0, 0, 0);
-    schedulerHandler.taskAllocated(0, mockTaskAttempt, lr, container);
-    assertEquals(2, mockEventHandler.events.size());
-    assertTrue(mockEventHandler.events.get(1) instanceof AMContainerEventAssignTA);
-    AMContainerEventAssignTA assignEvent =
-        (AMContainerEventAssignTA) mockEventHandler.events.get(1);
-    assertEquals(priority, assignEvent.getPriority());
-    assertEquals(mockAttemptId, assignEvent.getTaskAttemptId());
-  }
-
-  @Test (timeout = 5000)
-  public void testTaskBasedAffinity() throws Exception {
-    Configuration conf = new Configuration(false);
-    schedulerHandler.init(conf);
-    schedulerHandler.start();
-
-    TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
-    TezTaskAttemptID taId = mock(TezTaskAttemptID.class);
-    String affVertexName = "srcVertex";
-    int affTaskIndex = 1;
-    TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(affVertexName, affTaskIndex);
-    VertexImpl affVertex = mock(VertexImpl.class);
-    TaskImpl affTask = mock(TaskImpl.class);
-    TaskAttemptImpl affAttempt = mock(TaskAttemptImpl.class);
-    ContainerId affCId = mock(ContainerId.class);
-    when(affVertex.getTotalTasks()).thenReturn(2);
-    when(affVertex.getTask(affTaskIndex)).thenReturn(affTask);
-    when(affTask.getSuccessfulAttempt()).thenReturn(affAttempt);
-    when(affAttempt.getAssignedContainerID()).thenReturn(affCId);
-    when(mockAppContext.getCurrentDAG().getVertex(affVertexName)).thenReturn(affVertex);
-    Resource resource = Resource.newInstance(100, 1);
-    AMSchedulerEventTALaunchRequest event = new AMSchedulerEventTALaunchRequest
-        (taId, resource, null, mockTaskAttempt, locHint, 3, null, 0, 0, 0);
-    schedulerHandler.notify.set(false);
-    schedulerHandler.handle(event);
-    synchronized (schedulerHandler.notify) {
-      while (!schedulerHandler.notify.get()) {
-        schedulerHandler.notify.wait();
-      }
-    }
-    
-    // verify mockTaskAttempt affinitized to expected affCId
-    verify(mockTaskScheduler, times(1)).allocateTask(mockTaskAttempt, resource, affCId,
-        Priority.newInstance(3), null, event);
-    
-    schedulerHandler.stop();
-    schedulerHandler.close();
-  }
-  
-  @Test (timeout = 5000)
-  public void testContainerPreempted() throws IOException {
-    Configuration conf = new Configuration(false);
-    schedulerHandler.init(conf);
-    schedulerHandler.start();
-    
-    String diagnostics = "Container preempted by RM.";
-    TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
-    ContainerStatus mockStatus = mock(ContainerStatus.class);
-    ContainerId mockCId = mock(ContainerId.class);
-    AMContainer mockAMContainer = mock(AMContainer.class);
-    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
-    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
-    when(mockStatus.getContainerId()).thenReturn(mockCId);
-    when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
-    when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.PREEMPTED);
-    schedulerHandler.containerCompleted(0, mockTask, mockStatus);
-    assertEquals(1, mockEventHandler.events.size());
-    Event event = mockEventHandler.events.get(0);
-    assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
-    AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
-    assertEquals(mockCId, completedEvent.getContainerId());
-    assertEquals("Container preempted externally. Container preempted by RM.",
-        completedEvent.getDiagnostics());
-    assertTrue(completedEvent.isPreempted());
-    assertEquals(TaskAttemptTerminationCause.EXTERNAL_PREEMPTION,
-        completedEvent.getTerminationCause());
-    Assert.assertFalse(completedEvent.isDiskFailed());
-
-    schedulerHandler.stop();
-    schedulerHandler.close();
-  }
-  
-  @Test (timeout = 5000)
-  public void testContainerInternalPreempted() throws IOException {
-    Configuration conf = new Configuration(false);
-    schedulerHandler.init(conf);
-    schedulerHandler.start();
-
-    AMContainer mockAmContainer = mock(AMContainer.class);
-    when(mockAmContainer.getTaskSchedulerIdentifier()).thenReturn(0);
-    when(mockAmContainer.getContainerLauncherIdentifier()).thenReturn(0);
-    when(mockAmContainer.getTaskCommunicatorIdentifier()).thenReturn(0);
-    ContainerId mockCId = mock(ContainerId.class);
-    verify(mockTaskScheduler, times(0)).deallocateContainer((ContainerId) any());
-    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAmContainer);
-    schedulerHandler.preemptContainer(0, mockCId);
-    verify(mockTaskScheduler, times(1)).deallocateContainer(mockCId);
-    assertEquals(1, mockEventHandler.events.size());
-    Event event = mockEventHandler.events.get(0);
-    assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
-    AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
-    assertEquals(mockCId, completedEvent.getContainerId());
-    assertEquals("Container preempted internally", completedEvent.getDiagnostics());
-    assertTrue(completedEvent.isPreempted());
-    Assert.assertFalse(completedEvent.isDiskFailed());
-    assertEquals(TaskAttemptTerminationCause.INTERNAL_PREEMPTION,
-        completedEvent.getTerminationCause());
-
-    schedulerHandler.stop();
-    schedulerHandler.close();
-  }
-  
-  @Test (timeout = 5000)
-  public void testContainerDiskFailed() throws IOException {
-    Configuration conf = new Configuration(false);
-    schedulerHandler.init(conf);
-    schedulerHandler.start();
-    
-    String diagnostics = "NM disk failed.";
-    TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
-    ContainerStatus mockStatus = mock(ContainerStatus.class);
-    ContainerId mockCId = mock(ContainerId.class);
-    AMContainer mockAMContainer = mock(AMContainer.class);
-    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
-    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
-    when(mockStatus.getContainerId()).thenReturn(mockCId);
-    when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
-    when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.DISKS_FAILED);
-    schedulerHandler.containerCompleted(0, mockTask, mockStatus);
-    assertEquals(1, mockEventHandler.events.size());
-    Event event = mockEventHandler.events.get(0);
-    assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
-    AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
-    assertEquals(mockCId, completedEvent.getContainerId());
-    assertEquals("Container disk failed. NM disk failed.",
-        completedEvent.getDiagnostics());
-    Assert.assertFalse(completedEvent.isPreempted());
-    assertTrue(completedEvent.isDiskFailed());
-    assertEquals(TaskAttemptTerminationCause.NODE_DISK_ERROR,
-        completedEvent.getTerminationCause());
-
-    schedulerHandler.stop();
-    schedulerHandler.close();
-  }
-
-  @Test (timeout = 5000)
-  public void testContainerExceededPMem() throws IOException {
-    Configuration conf = new Configuration(false);
-    schedulerHandler.init(conf);
-    schedulerHandler.start();
-
-    String diagnostics = "Exceeded Physical Memory";
-    TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
-    ContainerStatus mockStatus = mock(ContainerStatus.class);
-    ContainerId mockCId = mock(ContainerId.class);
-    AMContainer mockAMContainer = mock(AMContainer.class);
-    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
-    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
-    when(mockStatus.getContainerId()).thenReturn(mockCId);
-    when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
-    // use -104 rather than ContainerExitStatus.KILLED_EXCEEDED_PMEM because
-    // ContainerExitStatus.KILLED_EXCEEDED_PMEM is only available after hadoop-2.5
-    when(mockStatus.getExitStatus()).thenReturn(-104);
-    schedulerHandler.containerCompleted(0, mockTask, mockStatus);
-    assertEquals(1, mockEventHandler.events.size());
-    Event event = mockEventHandler.events.get(0);
-    assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
-    AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
-    assertEquals(mockCId, completedEvent.getContainerId());
-    assertEquals("Container failed, exitCode=-104. Exceeded Physical Memory",
-        completedEvent.getDiagnostics());
-    Assert.assertFalse(completedEvent.isPreempted());
-    Assert.assertFalse(completedEvent.isDiskFailed());
-    assertEquals(TaskAttemptTerminationCause.CONTAINER_EXITED,
-        completedEvent.getTerminationCause());
-
-    schedulerHandler.stop();
-    schedulerHandler.close();
-  }
-
-  @Test (timeout = 5000)
-  public void testHistoryUrlConf() throws Exception {
-    Configuration conf = schedulerHandler.appContext.getAMConf();
-
-    // ensure history url is empty when timeline server is not the logging class
-    conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9999");
-    assertTrue("".equals(schedulerHandler.getHistoryUrl()));
-
-    // ensure expansion of url happens
-    conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
-        "org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService");
-    final ApplicationId mockApplicationId = mock(ApplicationId.class);
-    doReturn("TEST_APP_ID").when(mockApplicationId).toString();
-    doReturn(mockApplicationId).when(mockAppContext).getApplicationID();
-    assertTrue("http://ui-host:9999/#/tez-app/TEST_APP_ID"
-        .equals(schedulerHandler.getHistoryUrl()));
-
-    // ensure the trailing / in history url is handled
-    conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9998/");
-    assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
-        .equals(schedulerHandler.getHistoryUrl()));
-
-    // ensure missing scheme in history url is handled
-    conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/");
-    Assert.assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
-        .equals(schedulerHandler.getHistoryUrl()));
-
-    // handle bad template ex without begining /
-    conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
-        "__HISTORY_URL_BASE__#/somepath");
-    assertTrue("http://ui-host:9998/#/somepath"
-        .equals(schedulerHandler.getHistoryUrl()));
-
-    conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
-        "__HISTORY_URL_BASE__?viewPath=tez-app/__APPLICATION_ID__");
-    conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://localhost/ui/tez");
-    assertTrue("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID"
-        .equals(schedulerHandler.getHistoryUrl()));
-
-  }
-
-  @Test(timeout = 5000)
-  public void testNoSchedulerSpecified() throws IOException {
-    try {
-      new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
-          mockSigMatcher, mockWebUIService, null, false);
-      fail("Expecting an IllegalStateException with no schedulers specified");
-    } catch (IllegalArgumentException e) {
-    }
-  }
-
-  // Verified via statics
-  @Test(timeout = 5000)
-  public void testCustomTaskSchedulerSetup() throws IOException {
-    Configuration conf = new Configuration(false);
-    conf.set("testkey", "testval");
-    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
-
-    String customSchedulerName = "fakeScheduler";
-    List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
-    ByteBuffer bb = ByteBuffer.allocate(4);
-    bb.putInt(0, 3);
-    UserPayload userPayload = UserPayload.create(bb);
-    taskSchedulers.add(
-        new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName())
-            .setUserPayload(userPayload));
-    taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
-        .setUserPayload(defaultPayload));
-
-    TSEHForMultipleSchedulersTest tseh =
-        new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
-            mockSigMatcher, mockWebUIService, taskSchedulers, false);
-
-    tseh.init(conf);
-    tseh.start();
-
-    // Verify that the YARN task scheduler is installed by default
-    assertTrue(tseh.getYarnSchedulerCreated());
-    assertFalse(tseh.getUberSchedulerCreated());
-    assertEquals(2, tseh.getNumCreateInvocations());
-
-    // Verify the order of the schedulers
-    assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
-    assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));
-
-    // Verify the payload setup for the custom task scheduler
-    assertNotNull(tseh.getTaskSchedulerContext(0));
-    assertEquals(bb, tseh.getTaskSchedulerContext(0).getInitialUserPayload().getPayload());
-
-    // Verify the payload on the yarn scheduler
-    assertNotNull(tseh.getTaskSchedulerContext(1));
-    Configuration parsed = TezUtils.createConfFromUserPayload(tseh.getTaskSchedulerContext(1).getInitialUserPayload());
-    assertEquals("testval", parsed.get("testkey"));
-  }
-
-  @Test(timeout = 5000)
-  public void testTaskSchedulerRouting() throws Exception {
-    Configuration conf = new Configuration(false);
-    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
-
-    String customSchedulerName = "fakeScheduler";
-    List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
-    ByteBuffer bb = ByteBuffer.allocate(4);
-    bb.putInt(0, 3);
-    UserPayload userPayload = UserPayload.create(bb);
-    taskSchedulers.add(
-        new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName())
-            .setUserPayload(userPayload));
-    taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
-        .setUserPayload(defaultPayload));
-
-    TSEHForMultipleSchedulersTest tseh =
-        new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
-            mockSigMatcher, mockWebUIService, taskSchedulers, false);
-
-    tseh.init(conf);
-    tseh.start();
-
-    // Verify that the YARN task scheduler is installed by default
-    assertTrue(tseh.getYarnSchedulerCreated());
-    assertFalse(tseh.getUberSchedulerCreated());
-    assertEquals(2, tseh.getNumCreateInvocations());
-
-    // Verify the order of the schedulers
-    assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
-    assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));
-
-    verify(tseh.getTestTaskScheduler(0)).initialize();
-    verify(tseh.getTestTaskScheduler(0)).start();
-
-    ApplicationId appId = ApplicationId.newInstance(1000, 1);
-    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
-    TezVertexID vertexID = TezVertexID.getInstance(dagId, 1);
-    TezTaskID taskId1 = TezTaskID.getInstance(vertexID, 1);
-    TezTaskAttemptID attemptId11 = TezTaskAttemptID.getInstance(taskId1, 1);
-    TezTaskID taskId2 = TezTaskID.getInstance(vertexID, 2);
-    TezTaskAttemptID attemptId21 = TezTaskAttemptID.getInstance(taskId2, 1);
-
-    Resource resource = Resource.newInstance(1024, 1);
-
-    TaskAttempt mockTaskAttempt1 = mock(TaskAttempt.class);
-    TaskAttempt mockTaskAttempt2 = mock(TaskAttempt.class);
-
-    AMSchedulerEventTALaunchRequest launchRequest1 =
-        new AMSchedulerEventTALaunchRequest(attemptId11, resource, mock(TaskSpec.class),
-            mockTaskAttempt1, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 0, 0,
-            0);
-
-    tseh.handle(launchRequest1);
-
-    verify(tseh.getTestTaskScheduler(0)).allocateTask(eq(mockTaskAttempt1), eq(resource),
-        any(String[].class), any(String[].class), any(Priority.class), any(Object.class),
-        eq(launchRequest1));
-
-    AMSchedulerEventTALaunchRequest launchRequest2 =
-        new AMSchedulerEventTALaunchRequest(attemptId21, resource, mock(TaskSpec.class),
-            mockTaskAttempt2, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 1, 0,
-            0);
-    tseh.handle(launchRequest2);
-    verify(tseh.getTestTaskScheduler(1)).allocateTask(eq(mockTaskAttempt2), eq(resource),
-        any(String[].class), any(String[].class), any(Priority.class), any(Object.class),
-        eq(launchRequest2));
-  }
-
-  private static class TSEHForMultipleSchedulersTest extends TaskSchedulerEventHandler {
-
-    private final TaskScheduler yarnTaskScheduler;
-    private final TaskScheduler uberTaskScheduler;
-    private final AtomicBoolean uberSchedulerCreated = new AtomicBoolean(false);
-    private final AtomicBoolean yarnSchedulerCreated = new AtomicBoolean(false);
-    private final AtomicInteger numCreateInvocations = new AtomicInteger(0);
-    private final Set<Integer> seenSchedulers = new HashSet<>();
-    private final List<TaskSchedulerContext> taskSchedulerContexts = new LinkedList<>();
-    private final List<String> taskSchedulerNames = new LinkedList<>();
-    private final List<TaskScheduler> testTaskSchedulers = new LinkedList<>();
-
-    public TSEHForMultipleSchedulersTest(AppContext appContext,
-                                         DAGClientServer clientService,
-                                         EventHandler eventHandler,
-                                         ContainerSignatureMatcher containerSignatureMatcher,
-                                         WebUIService webUI,
-                                         List<NamedEntityDescriptor> schedulerDescriptors,
-                                         boolean isPureLocalMode) {
-      super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI,
-          schedulerDescriptors, isPureLocalMode);
-      yarnTaskScheduler = mock(TaskScheduler.class);
-      uberTaskScheduler = mock(TaskScheduler.class);
-    }
-
-    @Override
-    TaskScheduler createTaskScheduler(String host, int port, String trackingUrl,
-                                      AppContext appContext,
-                                      NamedEntityDescriptor taskSchedulerDescriptor,
-                                      long customAppIdIdentifier,
-                                      int schedulerId) {
-
-      numCreateInvocations.incrementAndGet();
-      boolean added = seenSchedulers.add(schedulerId);
-      assertTrue("Cannot add multiple schedulers with the same schedulerId", added);
-      taskSchedulerNames.add(taskSchedulerDescriptor.getEntityName());
-      return super.createTaskScheduler(host, port, trackingUrl, appContext, taskSchedulerDescriptor,
-          customAppIdIdentifier, schedulerId);
-    }
-
-    @Override
-    TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
-      // Avoid wrapping in threads
-      return rawContext;
-    }
-
-    @Override
-    TaskScheduler createYarnTaskScheduler(TaskSchedulerContext taskSchedulerContext, int schedulerId) {
-      taskSchedulerContexts.add(taskSchedulerContext);
-      testTaskSchedulers.add(yarnTaskScheduler);
-      yarnSchedulerCreated.set(true);
-      return yarnTaskScheduler;
-    }
-
-    @Override
-    TaskScheduler createUberTaskScheduler(TaskSchedulerContext taskSchedulerContext, int schedulerId) {
-      taskSchedulerContexts.add(taskSchedulerContext);
-      uberSchedulerCreated.set(true);
-      testTaskSchedulers.add(yarnTaskScheduler);
-      return uberTaskScheduler;
-    }
-
-    @Override
-    TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext,
-                                            NamedEntityDescriptor taskSchedulerDescriptor, int schedulerId) {
-      taskSchedulerContexts.add(taskSchedulerContext);
-      TaskScheduler taskScheduler = spy(super.createCustomTaskScheduler(taskSchedulerContext, taskSchedulerDescriptor, schedulerId));
-      testTaskSchedulers.add(taskScheduler);
-      return taskScheduler;
-    }
-
-    @Override
-    // Inline handling of events.
-    public void handle(AMSchedulerEvent event) {
-      handleEvent(event);
-    }
-
-    public boolean getUberSchedulerCreated() {
-      return uberSchedulerCreated.get();
-    }
-
-    public boolean getYarnSchedulerCreated() {
-      return yarnSchedulerCreated.get();
-    }
-
-    public int getNumCreateInvocations() {
-      return numCreateInvocations.get();
-    }
-
-    public TaskSchedulerContext getTaskSchedulerContext(int schedulerId) {
-      return taskSchedulerContexts.get(schedulerId);
-    }
-
-    public String getTaskSchedulerName(int schedulerId) {
-      return taskSchedulerNames.get(schedulerId);
-    }
-
-    public TaskScheduler getTestTaskScheduler(int schedulerId) {
-      return testTaskSchedulers.get(schedulerId);
-    }
-  }
-
-  public static class FakeTaskScheduler extends TaskScheduler {
-
-    public FakeTaskScheduler(
-        TaskSchedulerContext taskSchedulerContext) {
-      super(taskSchedulerContext);
-    }
-
-    @Override
-    public Resource getAvailableResources() {
-      return null;
-    }
-
-    @Override
-    public int getClusterNodeCount() {
-      return 0;
-    }
-
-    @Override
-    public void dagComplete() {
-
-    }
-
-    @Override
-    public Resource getTotalResources() {
-      return null;
-    }
-
-    @Override
-    public void blacklistNode(NodeId nodeId) {
-
-    }
-
-    @Override
-    public void unblacklistNode(NodeId nodeId) {
-
-    }
-
-    @Override
-    public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks,
-                             Priority priority, Object containerSignature, Object clientCookie) {
-
-    }
-
-    @Override
-    public void allocateTask(Object task, Resource capability, ContainerId containerId,
-                             Priority priority, Object containerSignature, Object clientCookie) {
-
-    }
-
-    @Override
-    public boolean deallocateTask(Object task, boolean taskSucceeded,
-                                  TaskAttemptEndReason endReason,
-                                  String diagnostics) {
-      return false;
-    }
-
-    @Override
-    public Object deallocateContainer(ContainerId containerId) {
-      return null;
-    }
-
-    @Override
-    public void setShouldUnregister() {
-
-    }
-
-    @Override
-    public boolean hasUnregistered() {
-      return false;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
index c13ca5a..b1bc491 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
@@ -125,19 +125,19 @@ class TestTaskSchedulerHelpers {
   }
   
   // Overrides start / stop. Will be controlled without the extra event handling thread.
-  static class TaskSchedulerEventHandlerForTest extends
-      TaskSchedulerEventHandler {
+  static class TaskSchedulerManagerForTest extends
+      TaskSchedulerManager {
 
     private TezAMRMClientAsync<CookieContainerRequest> amrmClientAsync;
     private ContainerSignatureMatcher containerSignatureMatcher;
     private UserPayload defaultPayload;
 
     @SuppressWarnings("rawtypes")
-    public TaskSchedulerEventHandlerForTest(AppContext appContext,
-        EventHandler eventHandler,
-        TezAMRMClientAsync<CookieContainerRequest> amrmClientAsync,
-        ContainerSignatureMatcher containerSignatureMatcher,
-        UserPayload defaultPayload) {
+    public TaskSchedulerManagerForTest(AppContext appContext,
+                                       EventHandler eventHandler,
+                                       TezAMRMClientAsync<CookieContainerRequest> amrmClientAsync,
+                                       ContainerSignatureMatcher containerSignatureMatcher,
+                                       UserPayload defaultPayload) {
       super(appContext, null, eventHandler, containerSignatureMatcher, null,
           Lists.newArrayList(new NamedEntityDescriptor("FakeScheduler", null)),
           false);


[6/7] tez git commit: TEZ-2708. Rename classes and variables post TEZ-2003 changes. (sseth)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
deleted file mode 100644
index 07d269d..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ /dev/null
@@ -1,410 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.launcher;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
-import org.apache.tez.serviceplugins.api.ContainerLauncher;
-import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
-import org.apache.tez.serviceplugins.api.ContainerStopRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy;
-import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezUncheckedException;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-
-// TODO See what part of this lifecycle and state management can be simplified.
-// Ideally, no state - only sendStart / sendStop.
-
-// TODO Review this entire code and clean it up.
-
-/**
- * This class is responsible for launching of containers.
- */
-public class ContainerLauncherImpl extends ContainerLauncher {
-
-  // TODO Ensure the same thread is used to launch / stop the same container. Or - ensure event ordering.
-  static final Logger LOG = LoggerFactory.getLogger(ContainerLauncherImpl.class);
-
-  private final ConcurrentHashMap<ContainerId, Container> containers =
-    new ConcurrentHashMap<>();
-  protected ThreadPoolExecutor launcherPool;
-  protected static final int INITIAL_POOL_SIZE = 10;
-  private final int limitOnPoolSize;
-  private final Configuration conf;
-  private Thread eventHandlingThread;
-  protected BlockingQueue<ContainerOp> eventQueue = new LinkedBlockingQueue<>();
-  private ContainerManagementProtocolProxy cmProxy;
-  private AtomicBoolean serviceStopped = new AtomicBoolean(false);
-
-  private Container getContainer(ContainerOp event) {
-    ContainerId id = event.getBaseOperation().getContainerId();
-    Container c = containers.get(id);
-    if(c == null) {
-      c = new Container(id,
-          event.getBaseOperation().getNodeId().toString(), event.getBaseOperation().getContainerToken());
-      Container old = containers.putIfAbsent(id, c);
-      if(old != null) {
-        c = old;
-      }
-    }
-    return c;
-  }
-
-  private void removeContainerIfDone(ContainerId id) {
-    Container c = containers.get(id);
-    if(c != null && c.isCompletelyDone()) {
-      containers.remove(id);
-    }
-  }
-
-
-  private static enum ContainerState {
-    PREP, FAILED, RUNNING, DONE, KILLED_BEFORE_LAUNCH
-  }
-
-  private class Container {
-    private ContainerState state;
-    // store enough information to be able to cleanup the container
-    private ContainerId containerID;
-    final private String containerMgrAddress;
-    private Token containerToken;
-
-    public Container(ContainerId containerID,
-        String containerMgrAddress, Token containerToken) {
-      this.state = ContainerState.PREP;
-      this.containerMgrAddress = containerMgrAddress;
-      this.containerID = containerID;
-      this.containerToken = containerToken;
-    }
-
-    public synchronized boolean isCompletelyDone() {
-      return state == ContainerState.DONE || state == ContainerState.FAILED;
-    }
-
-    @SuppressWarnings("unchecked")
-    public synchronized void launch(ContainerLaunchRequest event) {
-      LOG.info("Launching Container with Id: " + event.getContainerId());
-      if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
-        state = ContainerState.DONE;
-        sendContainerLaunchFailedMsg(event.getContainerId(),
-            "Container was killed before it was launched");
-        return;
-      }
-
-      ContainerManagementProtocolProxyData proxy = null;
-      try {
-
-        proxy = getCMProxy(containerID, containerMgrAddress,
-            containerToken);
-
-        // Construct the actual Container
-        ContainerLaunchContext containerLaunchContext =
-          event.getContainerLaunchContext();
-
-        // Now launch the actual container
-        StartContainerRequest startRequest = Records
-          .newRecord(StartContainerRequest.class);
-        startRequest.setContainerToken(event.getContainerToken());
-        startRequest.setContainerLaunchContext(containerLaunchContext);
-
-        StartContainersResponse response =
-            proxy.getContainerManagementProtocol().startContainers(
-                StartContainersRequest.newInstance(
-                    Collections.singletonList(startRequest)));
-        if (response.getFailedRequests() != null
-            && !response.getFailedRequests().isEmpty()) {
-          throw response.getFailedRequests().get(containerID).deSerialize();
-        }
-
-        // after launching, send launched event to task attempt to move
-        // it from ASSIGNED to RUNNING state
-        getContext().containerLaunched(containerID);
-        this.state = ContainerState.RUNNING;
-      } catch (Throwable t) {
-        String message = "Container launch failed for " + containerID + " : "
-            + ExceptionUtils.getStackTrace(t);
-        this.state = ContainerState.FAILED;
-        sendContainerLaunchFailedMsg(containerID, message);
-      } finally {
-        if (proxy != null) {
-          cmProxy.mayBeCloseProxy(proxy);
-        }
-      }
-    }
-
-    @SuppressWarnings("unchecked")
-    public synchronized void kill() {
-
-      if(isCompletelyDone()) {
-        return;
-      }
-      if(this.state == ContainerState.PREP) {
-        this.state = ContainerState.KILLED_BEFORE_LAUNCH;
-      } else {
-        LOG.info("Sending a stop request to the NM for ContainerId: "
-            + containerID);
-
-        ContainerManagementProtocolProxyData proxy = null;
-        try {
-          proxy = getCMProxy(this.containerID, this.containerMgrAddress,
-              this.containerToken);
-
-            // kill the remote container if already launched
-            StopContainersRequest stopRequest = Records
-              .newRecord(StopContainersRequest.class);
-            stopRequest.setContainerIds(Collections.singletonList(containerID));
-
-            proxy.getContainerManagementProtocol().stopContainers(stopRequest);
-
-            // If stopContainer returns without an error, assuming the stop made
-            // it over to the NodeManager.
-          getContext().containerStopRequested(containerID);
-        } catch (Throwable t) {
-
-          // ignore the cleanup failure
-          String message = "cleanup failed for container "
-            + this.containerID + " : "
-            + ExceptionUtils.getStackTrace(t);
-          getContext().containerStopFailed(containerID, message);
-          LOG.warn(message);
-          this.state = ContainerState.DONE;
-          return;
-        } finally {
-          if (proxy != null) {
-            cmProxy.mayBeCloseProxy(proxy);
-          }
-        }
-        this.state = ContainerState.DONE;
-      }
-    }
-  }
-
-  public ContainerLauncherImpl(ContainerLauncherContext containerLauncherContext) {
-    super(containerLauncherContext);
-    try {
-      this.conf = TezUtils.createConfFromUserPayload(containerLauncherContext.getInitialUserPayload());
-    } catch (IOException e) {
-      throw new TezUncheckedException(
-          "Failed to parse user payload for " + ContainerLauncherImpl.class.getSimpleName(), e);
-    }
-    conf.setInt(
-        CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
-        0);
-    this.limitOnPoolSize = conf.getInt(
-        TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
-        TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT_DEFAULT);
-    LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize);
-  }
-
-  @Override
-  public void start() {
-    // pass a copy of config to ContainerManagementProtocolProxy until YARN-3497 is fixed
-    cmProxy =
-        new ContainerManagementProtocolProxy(conf);
-
-    ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
-        "ContainerLauncher #%d").setDaemon(true).build();
-
-    // Start with a default core-pool size of 10 and change it dynamically.
-    launcherPool = new ThreadPoolExecutor(INITIAL_POOL_SIZE,
-        Integer.MAX_VALUE, 1, TimeUnit.HOURS,
-        new LinkedBlockingQueue<Runnable>(),
-        tf, new CustomizedRejectedExecutionHandler());
-    eventHandlingThread = new Thread() {
-      @Override
-      public void run() {
-        ContainerOp event = null;
-        while (!Thread.currentThread().isInterrupted()) {
-          try {
-            event = eventQueue.take();
-          } catch (InterruptedException e) {
-            if(!serviceStopped.get()) {
-              LOG.error("Returning, interrupted : " + e);
-            }
-            return;
-          }
-          int poolSize = launcherPool.getCorePoolSize();
-
-          // See if we need up the pool size only if haven't reached the
-          // maximum limit yet.
-          if (poolSize != limitOnPoolSize) {
-
-            // nodes where containers will run at *this* point of time. This is
-            // *not* the cluster size and doesn't need to be.
-            int numNodes =
-                getContext().getNumNodes(TezConstants.getTezYarnServicePluginName());
-            int idealPoolSize = Math.min(limitOnPoolSize, numNodes);
-
-            if (poolSize < idealPoolSize) {
-              // Bump up the pool size to idealPoolSize+INITIAL_POOL_SIZE, the
-              // later is just a buffer so we are not always increasing the
-              // pool-size
-              int newPoolSize = Math.min(limitOnPoolSize, idealPoolSize
-                  + INITIAL_POOL_SIZE);
-              LOG.info("Setting ContainerLauncher pool size to " + newPoolSize
-                  + " as number-of-nodes to talk to is " + numNodes);
-              launcherPool.setCorePoolSize(newPoolSize);
-            }
-          }
-
-          // the events from the queue are handled in parallel
-          // using a thread pool
-          launcherPool.execute(createEventProcessor(event));
-
-          // TODO: Group launching of multiple containers to a single
-          // NodeManager into a single connection
-        }
-      }
-    };
-    eventHandlingThread.setName("ContainerLauncher Event Handler");
-    eventHandlingThread.start();
-  }
-
-  private void shutdownAllContainers() {
-    for (Container ct : this.containers.values()) {
-      if (ct != null) {
-        ct.kill();
-      }
-    }
-  }
-
-  @Override
-  public void shutdown() {
-    if(!serviceStopped.compareAndSet(false, true)) {
-      LOG.info("Ignoring multiple stops");
-      return;
-    }
-    // shutdown any containers that might be left running
-    shutdownAllContainers();
-    if (eventHandlingThread != null) {
-      eventHandlingThread.interrupt();
-    }
-    if (launcherPool != null) {
-      launcherPool.shutdownNow();
-    }
-  }
-
-  protected EventProcessor createEventProcessor(ContainerOp event) {
-    return new EventProcessor(event);
-  }
-
-  protected ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData getCMProxy(
-      ContainerId containerID, final String containerManagerBindAddr,
-      Token containerToken) throws IOException {
-    return cmProxy.getProxy(containerManagerBindAddr, containerID);
-  }
-
-  /**
-   * Setup and start the container on remote nodemanager.
-   */
-  class EventProcessor implements Runnable {
-    private ContainerOp event;
-
-    EventProcessor(ContainerOp event) {
-      this.event = event;
-    }
-
-    @Override
-    public void run() {
-      LOG.info("Processing operation {}", event.toString());
-
-      // Load ContainerManager tokens before creating a connection.
-      // TODO: Do it only once per NodeManager.
-      ContainerId containerID = event.getBaseOperation().getContainerId();
-
-      Container c = getContainer(event);
-      switch(event.getOpType()) {
-        case LAUNCH_REQUEST:
-          ContainerLaunchRequest launchRequest = event.getLaunchRequest();
-          c.launch(launchRequest);
-          break;
-        case STOP_REQUEST:
-          c.kill();
-          break;
-      }
-      removeContainerIfDone(containerID);
-    }
-  }
-
-  /**
-   * ThreadPoolExecutor.submit may fail if you are submitting task
-   * when ThreadPoolExecutor is shutting down (DAGAppMaster is shutting down).
-   * Use this CustomizedRejectedExecutionHandler to just logging rather than abort the application.
-   */
-  private static class CustomizedRejectedExecutionHandler implements RejectedExecutionHandler {
-    @Override
-    public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
-      LOG.warn("Can't submit task to ThreadPoolExecutor:" + executor);
-    }
-  }
-
-  @SuppressWarnings("unchecked")
-  void sendContainerLaunchFailedMsg(ContainerId containerId,
-      String message) {
-    LOG.error(message);
-    getContext().containerLaunchFailed(containerId, message);
-  }
-
-
-  @Override
-  public void launchContainer(ContainerLaunchRequest launchRequest) {
-    try {
-      eventQueue.put(new ContainerOp(ContainerOp.OPType.LAUNCH_REQUEST, launchRequest));
-    } catch (InterruptedException e) {
-      throw new TezUncheckedException(e);
-    }
-  }
-
-  @Override
-  public void stopContainer(ContainerStopRequest stopRequest) {
-    try {
-      eventQueue.put(new ContainerOp(ContainerOp.OPType.STOP_REQUEST, stopRequest));
-    } catch (InterruptedException e) {
-      throw new TezUncheckedException(e);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java
new file mode 100644
index 0000000..15a10bd
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.launcher;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.UnknownHostException;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
+import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
+import org.apache.tez.serviceplugins.api.ContainerLauncher;
+import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
+import org.apache.tez.serviceplugins.api.ContainerStopRequest;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.ContainerLauncherContextImpl;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.rm.ContainerLauncherEvent;
+import org.apache.tez.dag.app.rm.ContainerLauncherLaunchRequestEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ContainerLauncherManager extends AbstractService
+    implements EventHandler<ContainerLauncherEvent> {
+
+  static final Logger LOG = LoggerFactory.getLogger(TezContainerLauncherImpl.class);
+
+  @VisibleForTesting
+  final ContainerLauncher containerLaunchers[];
+  @VisibleForTesting
+  final ContainerLauncherContext containerLauncherContexts[];
+  protected final ServicePluginLifecycleAbstractService[] containerLauncherServiceWrappers;
+  private final AppContext appContext;
+
+  @VisibleForTesting
+  public ContainerLauncherManager(ContainerLauncher containerLauncher, AppContext context) {
+    super(ContainerLauncherManager.class.getName());
+    this.appContext = context;
+    containerLaunchers = new ContainerLauncher[] {containerLauncher};
+    containerLauncherContexts = new ContainerLauncherContext[] {containerLauncher.getContext()};
+    containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[]{
+        new ServicePluginLifecycleAbstractService<>(containerLauncher)};
+  }
+
+  // Accepting conf to setup final parameters, if required.
+  public ContainerLauncherManager(AppContext context,
+                                  TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+                                  String workingDirectory,
+                                  List<NamedEntityDescriptor> containerLauncherDescriptors,
+                                  boolean isPureLocalMode) {
+    super(ContainerLauncherManager.class.getName());
+
+    this.appContext = context;
+    Preconditions.checkArgument(
+        containerLauncherDescriptors != null && !containerLauncherDescriptors.isEmpty(),
+        "ContainerLauncherDescriptors must be specified");
+    containerLauncherContexts = new ContainerLauncherContext[containerLauncherDescriptors.size()];
+    containerLaunchers = new ContainerLauncher[containerLauncherDescriptors.size()];
+    containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[containerLauncherDescriptors.size()];
+
+
+    for (int i = 0; i < containerLauncherDescriptors.size(); i++) {
+      UserPayload userPayload = containerLauncherDescriptors.get(i).getUserPayload();
+      ContainerLauncherContext containerLauncherContext =
+          new ContainerLauncherContextImpl(context, taskCommunicatorManagerInterface, userPayload);
+      containerLauncherContexts[i] = containerLauncherContext;
+      containerLaunchers[i] = createContainerLauncher(containerLauncherDescriptors.get(i), context,
+          containerLauncherContext, taskCommunicatorManagerInterface, workingDirectory, i, isPureLocalMode);
+      containerLauncherServiceWrappers[i] = new ServicePluginLifecycleAbstractService<>(containerLaunchers[i]);
+    }
+  }
+
+  @VisibleForTesting
+  ContainerLauncher createContainerLauncher(
+      NamedEntityDescriptor containerLauncherDescriptor,
+      AppContext context,
+      ContainerLauncherContext containerLauncherContext,
+      TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+      String workingDirectory,
+      int containerLauncherIndex,
+      boolean isPureLocalMode) {
+    if (containerLauncherDescriptor.getEntityName().equals(
+        TezConstants.getTezYarnServicePluginName())) {
+      return createYarnContainerLauncher(containerLauncherContext);
+    } else if (containerLauncherDescriptor.getEntityName()
+        .equals(TezConstants.getTezUberServicePluginName())) {
+      return createUberContainerLauncher(containerLauncherContext, context,
+          taskCommunicatorManagerInterface,
+          workingDirectory, isPureLocalMode);
+    } else {
+      return createCustomContainerLauncher(containerLauncherContext, containerLauncherDescriptor);
+    }
+  }
+
+  @VisibleForTesting
+  ContainerLauncher createYarnContainerLauncher(ContainerLauncherContext containerLauncherContext) {
+    LOG.info("Creating DefaultContainerLauncher");
+    return new TezContainerLauncherImpl(containerLauncherContext);
+  }
+
+  @VisibleForTesting
+  ContainerLauncher createUberContainerLauncher(ContainerLauncherContext containerLauncherContext,
+                                                AppContext context,
+                                                TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+                                                String workingDirectory,
+                                                boolean isPureLocalMode) {
+    LOG.info("Creating LocalContainerLauncher");
+    // TODO Post TEZ-2003. LocalContainerLauncher is special cased, since it makes use of
+    // extensive internals which are only available at runtime. Will likely require
+    // some kind of runtime binding of parameters in the payload to work correctly.
+    try {
+      return
+          new LocalContainerLauncher(containerLauncherContext, context,
+              taskCommunicatorManagerInterface,
+              workingDirectory, isPureLocalMode);
+    } catch (UnknownHostException e) {
+      throw new TezUncheckedException(e);
+    }
+  }
+
+  @VisibleForTesting
+  @SuppressWarnings("unchecked")
+  ContainerLauncher createCustomContainerLauncher(ContainerLauncherContext containerLauncherContext,
+                                                  NamedEntityDescriptor containerLauncherDescriptor) {
+    LOG.info("Creating container launcher {}:{} ", containerLauncherDescriptor.getEntityName(),
+        containerLauncherDescriptor.getClassName());
+    Class<? extends ContainerLauncher> containerLauncherClazz =
+        (Class<? extends ContainerLauncher>) ReflectionUtils.getClazz(
+            containerLauncherDescriptor.getClassName());
+    try {
+      Constructor<? extends ContainerLauncher> ctor = containerLauncherClazz
+          .getConstructor(ContainerLauncherContext.class);
+      return ctor.newInstance(containerLauncherContext);
+    } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
+      throw new TezUncheckedException(e);
+    }
+
+  }
+
+  @Override
+  public void serviceInit(Configuration conf) {
+    for (int i = 0 ; i < containerLaunchers.length ; i++) {
+      containerLauncherServiceWrappers[i].init(conf);
+    }
+  }
+
+  @Override
+  public void serviceStart() {
+    for (int i = 0 ; i < containerLaunchers.length ; i++) {
+      containerLauncherServiceWrappers[i].start();
+    }
+  }
+
+  @Override
+  public void serviceStop() {
+    for (int i = 0 ; i < containerLaunchers.length ; i++) {
+      containerLauncherServiceWrappers[i].stop();
+    }
+  }
+
+  public void dagComplete(DAG dag) {
+    // Nothing required at the moment. Containers are shared across DAGs
+  }
+
+  public void dagSubmitted() {
+    // Nothing to do right now. Indicates that a new DAG has been submitted and
+    // the context has updated information.
+  }
+
+
+  @Override
+  public void handle(ContainerLauncherEvent event) {
+    int launcherId = event.getLauncherId();
+    String schedulerName = appContext.getTaskSchedulerName(event.getSchedulerId());
+    String taskCommName = appContext.getTaskCommunicatorName(event.getTaskCommId());
+    switch (event.getType()) {
+      case CONTAINER_LAUNCH_REQUEST:
+        ContainerLauncherLaunchRequestEvent launchEvent = (ContainerLauncherLaunchRequestEvent) event;
+        ContainerLaunchRequest launchRequest =
+            new ContainerLaunchRequest(launchEvent.getNodeId(), launchEvent.getContainerId(),
+                launchEvent.getContainerToken(), launchEvent.getContainerLaunchContext(),
+                launchEvent.getContainer(), schedulerName,
+                taskCommName);
+        containerLaunchers[launcherId].launchContainer(launchRequest);
+        break;
+      case CONTAINER_STOP_REQUEST:
+        ContainerStopRequest stopRequest =
+            new ContainerStopRequest(event.getNodeId(), event.getContainerId(),
+                event.getContainerToken(), schedulerName, taskCommName);
+        containerLaunchers[launcherId].stopContainer(stopRequest);
+        break;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
deleted file mode 100644
index b56bd5b..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.launcher;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.net.UnknownHostException;
-import java.util.List;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.common.ReflectionUtils;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
-import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
-import org.apache.tez.serviceplugins.api.ContainerLauncher;
-import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
-import org.apache.tez.serviceplugins.api.ContainerStopRequest;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.ContainerLauncherContextImpl;
-import org.apache.tez.dag.app.TaskAttemptListener;
-import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
-import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ContainerLauncherRouter extends AbstractService
-    implements EventHandler<NMCommunicatorEvent> {
-
-  static final Logger LOG = LoggerFactory.getLogger(ContainerLauncherImpl.class);
-
-  @VisibleForTesting
-  final ContainerLauncher containerLaunchers[];
-  @VisibleForTesting
-  final ContainerLauncherContext containerLauncherContexts[];
-  protected final ServicePluginLifecycleAbstractService[] containerLauncherServiceWrappers;
-  private final AppContext appContext;
-
-  @VisibleForTesting
-  public ContainerLauncherRouter(ContainerLauncher containerLauncher, AppContext context) {
-    super(ContainerLauncherRouter.class.getName());
-    this.appContext = context;
-    containerLaunchers = new ContainerLauncher[] {containerLauncher};
-    containerLauncherContexts = new ContainerLauncherContext[] {containerLauncher.getContext()};
-    containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[]{
-        new ServicePluginLifecycleAbstractService<>(containerLauncher)};
-  }
-
-  // Accepting conf to setup final parameters, if required.
-  public ContainerLauncherRouter(AppContext context,
-                                 TaskAttemptListener taskAttemptListener,
-                                 String workingDirectory,
-                                 List<NamedEntityDescriptor> containerLauncherDescriptors,
-                                 boolean isPureLocalMode) {
-    super(ContainerLauncherRouter.class.getName());
-
-    this.appContext = context;
-    Preconditions.checkArgument(
-        containerLauncherDescriptors != null && !containerLauncherDescriptors.isEmpty(),
-        "ContainerLauncherDescriptors must be specified");
-    containerLauncherContexts = new ContainerLauncherContext[containerLauncherDescriptors.size()];
-    containerLaunchers = new ContainerLauncher[containerLauncherDescriptors.size()];
-    containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[containerLauncherDescriptors.size()];
-
-
-    for (int i = 0; i < containerLauncherDescriptors.size(); i++) {
-      UserPayload userPayload = containerLauncherDescriptors.get(i).getUserPayload();
-      ContainerLauncherContext containerLauncherContext =
-          new ContainerLauncherContextImpl(context, taskAttemptListener, userPayload);
-      containerLauncherContexts[i] = containerLauncherContext;
-      containerLaunchers[i] = createContainerLauncher(containerLauncherDescriptors.get(i), context,
-          containerLauncherContext, taskAttemptListener, workingDirectory, i, isPureLocalMode);
-      containerLauncherServiceWrappers[i] = new ServicePluginLifecycleAbstractService<>(containerLaunchers[i]);
-    }
-  }
-
-  @VisibleForTesting
-  ContainerLauncher createContainerLauncher(
-      NamedEntityDescriptor containerLauncherDescriptor,
-      AppContext context,
-      ContainerLauncherContext containerLauncherContext,
-      TaskAttemptListener taskAttemptListener,
-      String workingDirectory,
-      int containerLauncherIndex,
-      boolean isPureLocalMode) {
-    if (containerLauncherDescriptor.getEntityName().equals(
-        TezConstants.getTezYarnServicePluginName())) {
-      return createYarnContainerLauncher(containerLauncherContext);
-    } else if (containerLauncherDescriptor.getEntityName()
-        .equals(TezConstants.getTezUberServicePluginName())) {
-      return createUberContainerLauncher(containerLauncherContext, context, taskAttemptListener,
-          workingDirectory, isPureLocalMode);
-    } else {
-      return createCustomContainerLauncher(containerLauncherContext, containerLauncherDescriptor);
-    }
-  }
-
-  @VisibleForTesting
-  ContainerLauncher createYarnContainerLauncher(ContainerLauncherContext containerLauncherContext) {
-    LOG.info("Creating DefaultContainerLauncher");
-    return new ContainerLauncherImpl(containerLauncherContext);
-  }
-
-  @VisibleForTesting
-  ContainerLauncher createUberContainerLauncher(ContainerLauncherContext containerLauncherContext,
-                                                AppContext context,
-                                                TaskAttemptListener taskAttemptListener,
-                                                String workingDirectory,
-                                                boolean isPureLocalMode) {
-    LOG.info("Creating LocalContainerLauncher");
-    // TODO Post TEZ-2003. LocalContainerLauncher is special cased, since it makes use of
-    // extensive internals which are only available at runtime. Will likely require
-    // some kind of runtime binding of parameters in the payload to work correctly.
-    try {
-      return
-          new LocalContainerLauncher(containerLauncherContext, context, taskAttemptListener,
-              workingDirectory, isPureLocalMode);
-    } catch (UnknownHostException e) {
-      throw new TezUncheckedException(e);
-    }
-  }
-
-  @VisibleForTesting
-  @SuppressWarnings("unchecked")
-  ContainerLauncher createCustomContainerLauncher(ContainerLauncherContext containerLauncherContext,
-                                                  NamedEntityDescriptor containerLauncherDescriptor) {
-    LOG.info("Creating container launcher {}:{} ", containerLauncherDescriptor.getEntityName(),
-        containerLauncherDescriptor.getClassName());
-    Class<? extends ContainerLauncher> containerLauncherClazz =
-        (Class<? extends ContainerLauncher>) ReflectionUtils.getClazz(
-            containerLauncherDescriptor.getClassName());
-    try {
-      Constructor<? extends ContainerLauncher> ctor = containerLauncherClazz
-          .getConstructor(ContainerLauncherContext.class);
-      return ctor.newInstance(containerLauncherContext);
-    } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
-      throw new TezUncheckedException(e);
-    }
-
-  }
-
-  @Override
-  public void serviceInit(Configuration conf) {
-    for (int i = 0 ; i < containerLaunchers.length ; i++) {
-      containerLauncherServiceWrappers[i].init(conf);
-    }
-  }
-
-  @Override
-  public void serviceStart() {
-    for (int i = 0 ; i < containerLaunchers.length ; i++) {
-      containerLauncherServiceWrappers[i].start();
-    }
-  }
-
-  @Override
-  public void serviceStop() {
-    for (int i = 0 ; i < containerLaunchers.length ; i++) {
-      containerLauncherServiceWrappers[i].stop();
-    }
-  }
-
-  public void dagComplete(DAG dag) {
-    // Nothing required at the moment. Containers are shared across DAGs
-  }
-
-  public void dagSubmitted() {
-    // Nothing to do right now. Indicates that a new DAG has been submitted and
-    // the context has updated information.
-  }
-
-
-  @Override
-  public void handle(NMCommunicatorEvent event) {
-    int launcherId = event.getLauncherId();
-    String schedulerName = appContext.getTaskSchedulerName(event.getSchedulerId());
-    String taskCommName = appContext.getTaskCommunicatorName(event.getTaskCommId());
-    switch (event.getType()) {
-      case CONTAINER_LAUNCH_REQUEST:
-        NMCommunicatorLaunchRequestEvent launchEvent = (NMCommunicatorLaunchRequestEvent) event;
-        ContainerLaunchRequest launchRequest =
-            new ContainerLaunchRequest(launchEvent.getNodeId(), launchEvent.getContainerId(),
-                launchEvent.getContainerToken(), launchEvent.getContainerLaunchContext(),
-                launchEvent.getContainer(), schedulerName,
-                taskCommName);
-        containerLaunchers[launcherId].launchContainer(launchRequest);
-        break;
-      case CONTAINER_STOP_REQUEST:
-        ContainerStopRequest stopRequest =
-            new ContainerStopRequest(event.getNodeId(), event.getContainerId(),
-                event.getContainerToken(), schedulerName, taskCommName);
-        containerLaunchers[launcherId].stopContainer(stopRequest);
-        break;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 1d3e6df..6cd6fce 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -63,7 +63,7 @@ import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
 import org.apache.tez.runtime.api.ExecutionContext;
 import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
@@ -83,7 +83,7 @@ public class LocalContainerLauncher extends ContainerLauncher {
   private final AppContext context;
   private final AtomicBoolean serviceStopped = new AtomicBoolean(false);
   private final String workingDirectory;
-  private final TaskAttemptListener tal;
+  private final TaskCommunicatorManagerInterface tal;
   private final Map<String, String> localEnv;
   private final ExecutionContext executionContext;
   private final int numExecutors;
@@ -105,7 +105,7 @@ public class LocalContainerLauncher extends ContainerLauncher {
 
   public LocalContainerLauncher(ContainerLauncherContext containerLauncherContext,
                                 AppContext context,
-                                TaskAttemptListener taskAttemptListener,
+                                TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
                                 String workingDirectory,
                                 boolean isPureLocalMode) throws UnknownHostException {
     // TODO Post TEZ-2003. Most of this information is dynamic and only available after the AM
@@ -114,7 +114,7 @@ public class LocalContainerLauncher extends ContainerLauncher {
     // after the AM starts up.
     super(containerLauncherContext);
     this.context = context;
-    this.tal = taskAttemptListener;
+    this.tal = taskCommunicatorManagerInterface;
     this.workingDirectory = workingDirectory;
     this.isPureLocalMode = isPureLocalMode;
     if (isPureLocalMode) {

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java
new file mode 100644
index 0000000..3556c51
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java
@@ -0,0 +1,410 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.app.launcher;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
+import org.apache.tez.serviceplugins.api.ContainerLauncher;
+import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
+import org.apache.tez.serviceplugins.api.ContainerStopRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy;
+import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezUncheckedException;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+
+// TODO See what part of this lifecycle and state management can be simplified.
+// Ideally, no state - only sendStart / sendStop.
+
+// TODO Review this entire code and clean it up.
+
+/**
+ * This class is responsible for launching of containers.
+ */
+public class TezContainerLauncherImpl extends ContainerLauncher {
+
+  // TODO Ensure the same thread is used to launch / stop the same container. Or - ensure event ordering.
+  static final Logger LOG = LoggerFactory.getLogger(TezContainerLauncherImpl.class);
+
+  private final ConcurrentHashMap<ContainerId, Container> containers =
+    new ConcurrentHashMap<>();
+  protected ThreadPoolExecutor launcherPool;
+  protected static final int INITIAL_POOL_SIZE = 10;
+  private final int limitOnPoolSize;
+  private final Configuration conf;
+  private Thread eventHandlingThread;
+  protected BlockingQueue<ContainerOp> eventQueue = new LinkedBlockingQueue<>();
+  private ContainerManagementProtocolProxy cmProxy;
+  private AtomicBoolean serviceStopped = new AtomicBoolean(false);
+
+  private Container getContainer(ContainerOp event) {
+    ContainerId id = event.getBaseOperation().getContainerId();
+    Container c = containers.get(id);
+    if(c == null) {
+      c = new Container(id,
+          event.getBaseOperation().getNodeId().toString(), event.getBaseOperation().getContainerToken());
+      Container old = containers.putIfAbsent(id, c);
+      if(old != null) {
+        c = old;
+      }
+    }
+    return c;
+  }
+
+  private void removeContainerIfDone(ContainerId id) {
+    Container c = containers.get(id);
+    if(c != null && c.isCompletelyDone()) {
+      containers.remove(id);
+    }
+  }
+
+
+  private static enum ContainerState {
+    PREP, FAILED, RUNNING, DONE, KILLED_BEFORE_LAUNCH
+  }
+
+  private class Container {
+    private ContainerState state;
+    // store enough information to be able to cleanup the container
+    private ContainerId containerID;
+    final private String containerMgrAddress;
+    private Token containerToken;
+
+    public Container(ContainerId containerID,
+        String containerMgrAddress, Token containerToken) {
+      this.state = ContainerState.PREP;
+      this.containerMgrAddress = containerMgrAddress;
+      this.containerID = containerID;
+      this.containerToken = containerToken;
+    }
+
+    public synchronized boolean isCompletelyDone() {
+      return state == ContainerState.DONE || state == ContainerState.FAILED;
+    }
+
+    @SuppressWarnings("unchecked")
+    public synchronized void launch(ContainerLaunchRequest event) {
+      LOG.info("Launching Container with Id: " + event.getContainerId());
+      if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
+        state = ContainerState.DONE;
+        sendContainerLaunchFailedMsg(event.getContainerId(),
+            "Container was killed before it was launched");
+        return;
+      }
+
+      ContainerManagementProtocolProxyData proxy = null;
+      try {
+
+        proxy = getCMProxy(containerID, containerMgrAddress,
+            containerToken);
+
+        // Construct the actual Container
+        ContainerLaunchContext containerLaunchContext =
+          event.getContainerLaunchContext();
+
+        // Now launch the actual container
+        StartContainerRequest startRequest = Records
+          .newRecord(StartContainerRequest.class);
+        startRequest.setContainerToken(event.getContainerToken());
+        startRequest.setContainerLaunchContext(containerLaunchContext);
+
+        StartContainersResponse response =
+            proxy.getContainerManagementProtocol().startContainers(
+                StartContainersRequest.newInstance(
+                    Collections.singletonList(startRequest)));
+        if (response.getFailedRequests() != null
+            && !response.getFailedRequests().isEmpty()) {
+          throw response.getFailedRequests().get(containerID).deSerialize();
+        }
+
+        // after launching, send launched event to task attempt to move
+        // it from ASSIGNED to RUNNING state
+        getContext().containerLaunched(containerID);
+        this.state = ContainerState.RUNNING;
+      } catch (Throwable t) {
+        String message = "Container launch failed for " + containerID + " : "
+            + ExceptionUtils.getStackTrace(t);
+        this.state = ContainerState.FAILED;
+        sendContainerLaunchFailedMsg(containerID, message);
+      } finally {
+        if (proxy != null) {
+          cmProxy.mayBeCloseProxy(proxy);
+        }
+      }
+    }
+
+    @SuppressWarnings("unchecked")
+    public synchronized void kill() {
+
+      if(isCompletelyDone()) {
+        return;
+      }
+      if(this.state == ContainerState.PREP) {
+        this.state = ContainerState.KILLED_BEFORE_LAUNCH;
+      } else {
+        LOG.info("Sending a stop request to the NM for ContainerId: "
+            + containerID);
+
+        ContainerManagementProtocolProxyData proxy = null;
+        try {
+          proxy = getCMProxy(this.containerID, this.containerMgrAddress,
+              this.containerToken);
+
+            // kill the remote container if already launched
+            StopContainersRequest stopRequest = Records
+              .newRecord(StopContainersRequest.class);
+            stopRequest.setContainerIds(Collections.singletonList(containerID));
+
+            proxy.getContainerManagementProtocol().stopContainers(stopRequest);
+
+            // If stopContainer returns without an error, assuming the stop made
+            // it over to the NodeManager.
+          getContext().containerStopRequested(containerID);
+        } catch (Throwable t) {
+
+          // ignore the cleanup failure
+          String message = "cleanup failed for container "
+            + this.containerID + " : "
+            + ExceptionUtils.getStackTrace(t);
+          getContext().containerStopFailed(containerID, message);
+          LOG.warn(message);
+          this.state = ContainerState.DONE;
+          return;
+        } finally {
+          if (proxy != null) {
+            cmProxy.mayBeCloseProxy(proxy);
+          }
+        }
+        this.state = ContainerState.DONE;
+      }
+    }
+  }
+
+  public TezContainerLauncherImpl(ContainerLauncherContext containerLauncherContext) {
+    super(containerLauncherContext);
+    try {
+      this.conf = TezUtils.createConfFromUserPayload(containerLauncherContext.getInitialUserPayload());
+    } catch (IOException e) {
+      throw new TezUncheckedException(
+          "Failed to parse user payload for " + TezContainerLauncherImpl.class.getSimpleName(), e);
+    }
+    conf.setInt(
+        CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
+        0);
+    this.limitOnPoolSize = conf.getInt(
+        TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
+        TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT_DEFAULT);
+    LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize);
+  }
+
+  @Override
+  public void start() {
+    // pass a copy of config to ContainerManagementProtocolProxy until YARN-3497 is fixed
+    cmProxy =
+        new ContainerManagementProtocolProxy(conf);
+
+    ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
+        "ContainerLauncher #%d").setDaemon(true).build();
+
+    // Start with a default core-pool size of 10 and change it dynamically.
+    launcherPool = new ThreadPoolExecutor(INITIAL_POOL_SIZE,
+        Integer.MAX_VALUE, 1, TimeUnit.HOURS,
+        new LinkedBlockingQueue<Runnable>(),
+        tf, new CustomizedRejectedExecutionHandler());
+    eventHandlingThread = new Thread() {
+      @Override
+      public void run() {
+        ContainerOp event = null;
+        while (!Thread.currentThread().isInterrupted()) {
+          try {
+            event = eventQueue.take();
+          } catch (InterruptedException e) {
+            if(!serviceStopped.get()) {
+              LOG.error("Returning, interrupted : " + e);
+            }
+            return;
+          }
+          int poolSize = launcherPool.getCorePoolSize();
+
+          // See if we need up the pool size only if haven't reached the
+          // maximum limit yet.
+          if (poolSize != limitOnPoolSize) {
+
+            // nodes where containers will run at *this* point of time. This is
+            // *not* the cluster size and doesn't need to be.
+            int numNodes =
+                getContext().getNumNodes(TezConstants.getTezYarnServicePluginName());
+            int idealPoolSize = Math.min(limitOnPoolSize, numNodes);
+
+            if (poolSize < idealPoolSize) {
+              // Bump up the pool size to idealPoolSize+INITIAL_POOL_SIZE, the
+              // later is just a buffer so we are not always increasing the
+              // pool-size
+              int newPoolSize = Math.min(limitOnPoolSize, idealPoolSize
+                  + INITIAL_POOL_SIZE);
+              LOG.info("Setting ContainerLauncher pool size to " + newPoolSize
+                  + " as number-of-nodes to talk to is " + numNodes);
+              launcherPool.setCorePoolSize(newPoolSize);
+            }
+          }
+
+          // the events from the queue are handled in parallel
+          // using a thread pool
+          launcherPool.execute(createEventProcessor(event));
+
+          // TODO: Group launching of multiple containers to a single
+          // NodeManager into a single connection
+        }
+      }
+    };
+    eventHandlingThread.setName("ContainerLauncher Event Handler");
+    eventHandlingThread.start();
+  }
+
+  private void shutdownAllContainers() {
+    for (Container ct : this.containers.values()) {
+      if (ct != null) {
+        ct.kill();
+      }
+    }
+  }
+
+  @Override
+  public void shutdown() {
+    if(!serviceStopped.compareAndSet(false, true)) {
+      LOG.info("Ignoring multiple stops");
+      return;
+    }
+    // shutdown any containers that might be left running
+    shutdownAllContainers();
+    if (eventHandlingThread != null) {
+      eventHandlingThread.interrupt();
+    }
+    if (launcherPool != null) {
+      launcherPool.shutdownNow();
+    }
+  }
+
+  protected EventProcessor createEventProcessor(ContainerOp event) {
+    return new EventProcessor(event);
+  }
+
+  protected ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData getCMProxy(
+      ContainerId containerID, final String containerManagerBindAddr,
+      Token containerToken) throws IOException {
+    return cmProxy.getProxy(containerManagerBindAddr, containerID);
+  }
+
+  /**
+   * Setup and start the container on remote nodemanager.
+   */
+  class EventProcessor implements Runnable {
+    private ContainerOp event;
+
+    EventProcessor(ContainerOp event) {
+      this.event = event;
+    }
+
+    @Override
+    public void run() {
+      LOG.info("Processing operation {}", event.toString());
+
+      // Load ContainerManager tokens before creating a connection.
+      // TODO: Do it only once per NodeManager.
+      ContainerId containerID = event.getBaseOperation().getContainerId();
+
+      Container c = getContainer(event);
+      switch(event.getOpType()) {
+        case LAUNCH_REQUEST:
+          ContainerLaunchRequest launchRequest = event.getLaunchRequest();
+          c.launch(launchRequest);
+          break;
+        case STOP_REQUEST:
+          c.kill();
+          break;
+      }
+      removeContainerIfDone(containerID);
+    }
+  }
+
+  /**
+   * ThreadPoolExecutor.submit may fail if you are submitting task
+   * when ThreadPoolExecutor is shutting down (DAGAppMaster is shutting down).
+   * Use this CustomizedRejectedExecutionHandler to just logging rather than abort the application.
+   */
+  private static class CustomizedRejectedExecutionHandler implements RejectedExecutionHandler {
+    @Override
+    public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
+      LOG.warn("Can't submit task to ThreadPoolExecutor:" + executor);
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  void sendContainerLaunchFailedMsg(ContainerId containerId,
+      String message) {
+    LOG.error(message);
+    getContext().containerLaunchFailed(containerId, message);
+  }
+
+
+  @Override
+  public void launchContainer(ContainerLaunchRequest launchRequest) {
+    try {
+      eventQueue.put(new ContainerOp(ContainerOp.OPType.LAUNCH_REQUEST, launchRequest));
+    } catch (InterruptedException e) {
+      throw new TezUncheckedException(e);
+    }
+  }
+
+  @Override
+  public void stopContainer(ContainerStopRequest stopRequest) {
+    try {
+      eventQueue.put(new ContainerOp(ContainerOp.OPType.STOP_REQUEST, stopRequest));
+    } catch (InterruptedException e) {
+      throw new TezUncheckedException(e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java
new file mode 100644
index 0000000..add3254
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+public class ContainerLauncherEvent extends AbstractEvent<ContainerLauncherEventType> {
+
+  private final ContainerId containerId;
+  private final NodeId nodeId;
+  private final Token containerToken;
+  private final int launcherId;
+  private final int schedulerId;
+  private final int taskCommId;
+
+  public ContainerLauncherEvent(ContainerId containerId, NodeId nodeId,
+                                Token containerToken, ContainerLauncherEventType type,
+                                int launcherId,
+                                int schedulerId, int taskCommId) {
+    super(type);
+    this.containerId = containerId;
+    this.nodeId = nodeId;
+    this.containerToken = containerToken;
+    this.launcherId = launcherId;
+    this.schedulerId = schedulerId;
+    this.taskCommId = taskCommId;
+  }
+
+  public ContainerId getContainerId() {
+    return this.containerId;
+  }
+
+  public NodeId getNodeId() {
+    return this.nodeId;
+  }
+
+  public Token getContainerToken() {
+    return this.containerToken;
+  }
+
+  public int getLauncherId() {
+    return launcherId;
+  }
+
+  public int getSchedulerId() {
+    return schedulerId;
+  }
+
+  public int getTaskCommId() {
+    return taskCommId;
+  }
+
+  public String toSrting() {
+    return super.toString() + " for container " + containerId + ", nodeId: "
+        + nodeId + ", launcherId: " + launcherId + ", schedulerId=" + schedulerId +
+        ", taskCommId=" + taskCommId;
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result
+        + ((containerId == null) ? 0 : containerId.hashCode());
+    result = prime * result
+        + ((containerToken == null) ? 0 : containerToken.hashCode());
+    result = prime * result + ((nodeId == null) ? 0 : nodeId.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    ContainerLauncherEvent other = (ContainerLauncherEvent) obj;
+    if (containerId == null) {
+      if (other.containerId != null)
+        return false;
+    } else if (!containerId.equals(other.containerId))
+      return false;
+    if (containerToken == null) {
+      if (other.containerToken != null)
+        return false;
+    } else if (!containerToken.equals(other.containerToken))
+      return false;
+    if (nodeId == null) {
+      if (other.nodeId != null)
+        return false;
+    } else if (!nodeId.equals(other.nodeId))
+      return false;
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java
new file mode 100644
index 0000000..079509c
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+// TODO - Re-use the events in ContainerLauncher..
+public enum ContainerLauncherEventType {
+  CONTAINER_LAUNCH_REQUEST,
+  CONTAINER_STOP_REQUEST
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java
new file mode 100644
index 0000000..411451e
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+
+public class ContainerLauncherLaunchRequestEvent extends ContainerLauncherEvent {
+
+  private final ContainerLaunchContext clc;
+  private final Container container;
+  // The task communicator index for the specific container being launched.
+
+  public ContainerLauncherLaunchRequestEvent(ContainerLaunchContext clc,
+                                             Container container, int launcherId, int schedulerId,
+                                             int taskCommId) {
+    super(container.getId(), container.getNodeId(), container
+        .getContainerToken(), ContainerLauncherEventType.CONTAINER_LAUNCH_REQUEST,
+        launcherId, schedulerId, taskCommId);
+    this.clc = clc;
+    this.container = container;
+  }
+
+  public ContainerLaunchContext getContainerLaunchContext() {
+    return this.clc;
+  }
+
+  public Container getContainer() {
+    return container;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    if (!super.equals(o)) {
+      return false;
+    }
+
+    ContainerLauncherLaunchRequestEvent that = (ContainerLauncherLaunchRequestEvent) o;
+
+    if (clc != null ? !clc.equals(that.clc) : that.clc != null) {
+      return false;
+    }
+    if (container != null ? !container.equals(that.container) : that.container != null) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    int result = super.hashCode();
+    result = 7001 * result + (clc != null ? clc.hashCode() : 0);
+    result = 7001 * result + (container != null ? container.hashCode() : 0);
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java
new file mode 100644
index 0000000..69e7d30
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Token;
+
+public class ContainerLauncherStopRequestEvent extends ContainerLauncherEvent {
+
+  public ContainerLauncherStopRequestEvent(ContainerId containerId, NodeId nodeId,
+                                           Token containerToken, int launcherId, int schedulerId,
+                                           int taskCommId) {
+    super(containerId, nodeId, containerToken,
+        ContainerLauncherEventType.CONTAINER_STOP_REQUEST, launcherId, schedulerId, taskCommId);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
deleted file mode 100644
index dc50c37..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-
-public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType> {
-
-  private final ContainerId containerId;
-  private final NodeId nodeId;
-  private final Token containerToken;
-  private final int launcherId;
-  private final int schedulerId;
-  private final int taskCommId;
-
-  public NMCommunicatorEvent(ContainerId containerId, NodeId nodeId,
-                             Token containerToken, NMCommunicatorEventType type, int launcherId,
-                             int schedulerId, int taskCommId) {
-    super(type);
-    this.containerId = containerId;
-    this.nodeId = nodeId;
-    this.containerToken = containerToken;
-    this.launcherId = launcherId;
-    this.schedulerId = schedulerId;
-    this.taskCommId = taskCommId;
-  }
-
-  public ContainerId getContainerId() {
-    return this.containerId;
-  }
-
-  public NodeId getNodeId() {
-    return this.nodeId;
-  }
-
-  public Token getContainerToken() {
-    return this.containerToken;
-  }
-
-  public int getLauncherId() {
-    return launcherId;
-  }
-
-  public int getSchedulerId() {
-    return schedulerId;
-  }
-
-  public int getTaskCommId() {
-    return taskCommId;
-  }
-
-  public String toSrting() {
-    return super.toString() + " for container " + containerId + ", nodeId: "
-        + nodeId + ", launcherId: " + launcherId + ", schedulerId=" + schedulerId +
-        ", taskCommId=" + taskCommId;
-  }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result
-        + ((containerId == null) ? 0 : containerId.hashCode());
-    result = prime * result
-        + ((containerToken == null) ? 0 : containerToken.hashCode());
-    result = prime * result + ((nodeId == null) ? 0 : nodeId.hashCode());
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj)
-      return true;
-    if (obj == null)
-      return false;
-    if (getClass() != obj.getClass())
-      return false;
-    NMCommunicatorEvent other = (NMCommunicatorEvent) obj;
-    if (containerId == null) {
-      if (other.containerId != null)
-        return false;
-    } else if (!containerId.equals(other.containerId))
-      return false;
-    if (containerToken == null) {
-      if (other.containerToken != null)
-        return false;
-    } else if (!containerToken.equals(other.containerToken))
-      return false;
-    if (nodeId == null) {
-      if (other.nodeId != null)
-        return false;
-    } else if (!nodeId.equals(other.nodeId))
-      return false;
-    return true;
-  }
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEventType.java
deleted file mode 100644
index 9f3d989..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEventType.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-// TODO - Re-use the events in ContainerLauncher..
-public enum NMCommunicatorEventType {
-  CONTAINER_LAUNCH_REQUEST,
-  CONTAINER_STOP_REQUEST
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java
deleted file mode 100644
index c57b6be..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-
-public class NMCommunicatorLaunchRequestEvent extends NMCommunicatorEvent {
-
-  private final ContainerLaunchContext clc;
-  private final Container container;
-  // The task communicator index for the specific container being launched.
-
-  public NMCommunicatorLaunchRequestEvent(ContainerLaunchContext clc,
-      Container container, int launcherId, int schedulerId, int taskCommId) {
-    super(container.getId(), container.getNodeId(), container
-        .getContainerToken(), NMCommunicatorEventType.CONTAINER_LAUNCH_REQUEST,
-        launcherId, schedulerId, taskCommId);
-    this.clc = clc;
-    this.container = container;
-  }
-
-  public ContainerLaunchContext getContainerLaunchContext() {
-    return this.clc;
-  }
-
-  public Container getContainer() {
-    return container;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-    if (!super.equals(o)) {
-      return false;
-    }
-
-    NMCommunicatorLaunchRequestEvent that = (NMCommunicatorLaunchRequestEvent) o;
-
-    if (clc != null ? !clc.equals(that.clc) : that.clc != null) {
-      return false;
-    }
-    if (container != null ? !container.equals(that.container) : that.container != null) {
-      return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    int result = super.hashCode();
-    result = 7001 * result + (clc != null ? clc.hashCode() : 0);
-    result = 7001 * result + (container != null ? container.hashCode() : 0);
-    return result;
-  }
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
deleted file mode 100644
index 352f450..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Token;
-
-public class NMCommunicatorStopRequestEvent extends NMCommunicatorEvent {
-
-  public NMCommunicatorStopRequestEvent(ContainerId containerId, NodeId nodeId,
-      Token containerToken, int launcherId, int schedulerId, int taskCommId) {
-    super(containerId, nodeId, containerToken,
-        NMCommunicatorEventType.CONTAINER_STOP_REQUEST, launcherId, schedulerId, taskCommId);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java
index 2a9797f..37aa96b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java
@@ -33,7 +33,7 @@ import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
 
 public class TaskSchedulerContextImpl implements TaskSchedulerContext {
 
-  private final TaskSchedulerEventHandler tseh;
+  private final TaskSchedulerManager taskSchedulerManager;
   private final AppContext appContext;
   private final int schedulerId;
   private final String trackingUrl;
@@ -42,11 +42,11 @@ public class TaskSchedulerContextImpl implements TaskSchedulerContext {
   private final int clientPort;
   private final UserPayload initialUserPayload;
 
-  public TaskSchedulerContextImpl(TaskSchedulerEventHandler tseh, AppContext appContext,
+  public TaskSchedulerContextImpl(TaskSchedulerManager taskSchedulerManager, AppContext appContext,
                                   int schedulerId, String trackingUrl, long customClusterIdentifier,
                                   String appHostname, int clientPort,
                                   UserPayload initialUserPayload) {
-    this.tseh = tseh;
+    this.taskSchedulerManager = taskSchedulerManager;
     this.appContext = appContext;
     this.schedulerId = schedulerId;
     this.trackingUrl = trackingUrl;
@@ -62,54 +62,55 @@ public class TaskSchedulerContextImpl implements TaskSchedulerContext {
   // taskAllocated() upcall and deallocateTask() downcall
   @Override
   public void taskAllocated(Object task, Object appCookie, Container container) {
-    tseh.taskAllocated(schedulerId, task, appCookie, container);
+    taskSchedulerManager.taskAllocated(schedulerId, task, appCookie, container);
   }
 
   @Override
   public void containerCompleted(Object taskLastAllocated, ContainerStatus containerStatus) {
-    tseh.containerCompleted(schedulerId, taskLastAllocated, containerStatus);
+    taskSchedulerManager.containerCompleted(schedulerId, taskLastAllocated, containerStatus);
   }
 
   @Override
   public void containerBeingReleased(ContainerId containerId) {
-    tseh.containerBeingReleased(schedulerId, containerId);
+    taskSchedulerManager.containerBeingReleased(schedulerId, containerId);
   }
 
   @Override
   public void nodesUpdated(List<NodeReport> updatedNodes) {
-    tseh.nodesUpdated(schedulerId, updatedNodes);
+    taskSchedulerManager.nodesUpdated(schedulerId, updatedNodes);
   }
 
   @Override
   public void appShutdownRequested() {
-    tseh.appShutdownRequested(schedulerId);
+    taskSchedulerManager.appShutdownRequested(schedulerId);
   }
 
   @Override
   public void setApplicationRegistrationData(Resource maxContainerCapability,
                                              Map<ApplicationAccessType, String> appAcls,
                                              ByteBuffer clientAMSecretKey) {
-    tseh.setApplicationRegistrationData(schedulerId, maxContainerCapability, appAcls, clientAMSecretKey);
+    taskSchedulerManager.setApplicationRegistrationData(schedulerId, maxContainerCapability, appAcls,
+        clientAMSecretKey);
   }
 
   @Override
   public void onError(Throwable t) {
-    tseh.onError(schedulerId, t);
+    taskSchedulerManager.onError(schedulerId, t);
   }
 
   @Override
   public float getProgress() {
-    return tseh.getProgress(schedulerId);
+    return taskSchedulerManager.getProgress(schedulerId);
   }
 
   @Override
   public void preemptContainer(ContainerId containerId) {
-    tseh.preemptContainer(schedulerId, containerId);
+    taskSchedulerManager.preemptContainer(schedulerId, containerId);
   }
 
   @Override
   public AppFinalStatus getFinalAppStatus() {
-    return tseh.getFinalAppStatus();
+    return taskSchedulerManager.getFinalAppStatus();
   }
 
   @Override
@@ -130,7 +131,7 @@ public class TaskSchedulerContextImpl implements TaskSchedulerContext {
 
   @Override
   public ContainerSignatureMatcher getContainerSignatureMatcher() {
-    return tseh.getContainerSignatureMatcher();
+    return taskSchedulerManager.getContainerSignatureMatcher();
   }
 
   @Override


[5/7] tez git commit: TEZ-2708. Rename classes and variables post TEZ-2003 changes. (sseth)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
deleted file mode 100644
index 76d870c..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ /dev/null
@@ -1,784 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
-import org.apache.tez.serviceplugins.api.TaskScheduler;
-import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
-import org.apache.tez.serviceplugins.api.TaskSchedulerContext.AppFinalStatus;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.common.ReflectionUtils;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.TaskLocationHint;
-import org.apache.tez.dag.api.TaskLocationHint.TaskBasedLocationAffinity;
-import org.apache.tez.dag.api.client.DAGClientServer;
-import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
-import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.DAGAppMaster;
-import org.apache.tez.dag.app.DAGAppMasterState;
-import org.apache.tez.dag.app.dag.TaskAttempt;
-import org.apache.tez.dag.app.dag.Vertex;
-import org.apache.tez.dag.app.dag.event.DAGAppMasterEvent;
-import org.apache.tez.dag.app.dag.event.DAGAppMasterEventSchedulingServiceError;
-import org.apache.tez.dag.app.dag.event.DAGAppMasterEventType;
-import org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdateTAAssigned;
-import org.apache.tez.dag.app.rm.container.AMContainer;
-import org.apache.tez.dag.app.rm.container.AMContainerEventAssignTA;
-import org.apache.tez.dag.app.rm.container.AMContainerEventCompleted;
-import org.apache.tez.dag.app.rm.container.AMContainerEventLaunchRequest;
-import org.apache.tez.dag.app.rm.container.AMContainerEventStopRequest;
-import org.apache.tez.dag.app.rm.container.AMContainerEventTASucceeded;
-import org.apache.tez.dag.app.rm.container.AMContainerState;
-import org.apache.tez.common.ContainerSignatureMatcher;
-import org.apache.tez.dag.app.rm.node.AMNodeEventContainerAllocated;
-import org.apache.tez.dag.app.rm.node.AMNodeEventNodeCountUpdated;
-import org.apache.tez.dag.app.rm.node.AMNodeEventStateChanged;
-import org.apache.tez.dag.app.rm.node.AMNodeEventTaskAttemptEnded;
-import org.apache.tez.dag.app.rm.node.AMNodeEventTaskAttemptSucceeded;
-import org.apache.tez.dag.app.web.WebUIService;
-import org.apache.tez.dag.records.TaskAttemptTerminationCause;
-
-import com.google.common.base.Preconditions;
-
-
-public class TaskSchedulerEventHandler extends AbstractService implements
-                                               EventHandler<AMSchedulerEvent> {
-  static final Logger LOG = LoggerFactory.getLogger(TaskSchedulerEventHandler.class);
-
-  static final String APPLICATION_ID_PLACEHOLDER = "__APPLICATION_ID__";
-  static final String HISTORY_URL_BASE = "__HISTORY_URL_BASE__";
-
-  protected final AppContext appContext;
-  @SuppressWarnings("rawtypes")
-  private final EventHandler eventHandler;
-  private final String historyUrl;
-  private DAGAppMaster dagAppMaster;
-  private Map<ApplicationAccessType, String> appAcls = null;
-  private Thread eventHandlingThread;
-  private volatile boolean stopEventHandling;
-  // Has a signal (SIGTERM etc) been issued?
-  protected volatile boolean isSignalled = false;
-  final DAGClientServer clientService;
-  private final ContainerSignatureMatcher containerSignatureMatcher;
-  private int cachedNodeCount = -1;
-  private AtomicBoolean shouldUnregisterFlag =
-      new AtomicBoolean(false);
-  private final WebUIService webUI;
-  private final NamedEntityDescriptor[] taskSchedulerDescriptors;
-  protected final TaskScheduler[]taskSchedulers;
-  protected final ServicePluginLifecycleAbstractService []taskSchedulerServiceWrappers;
-
-  // Single executor service shared by all Schedulers for context callbacks
-  @VisibleForTesting
-  final ExecutorService appCallbackExecutor;
-
-  private final boolean isPureLocalMode;
-  // If running in non local-only mode, the YARN task scheduler will always run to take care of
-  // registration with YARN and heartbeats to YARN.
-  // Splitting registration and heartbeats is not straight-forward due to the taskScheduler being
-  // tied to a ContainerRequestType.
-  // Custom AppIds to avoid container conflicts if there's multiple sources
-  private final long SCHEDULER_APP_ID_BASE = 111101111;
-  private final long SCHEDULER_APP_ID_INCREMENT = 111111111;
-
-  BlockingQueue<AMSchedulerEvent> eventQueue
-                              = new LinkedBlockingQueue<AMSchedulerEvent>();
-
-  // Not tracking container / task to schedulerId. Instead relying on everything flowing through
-  // the system and being propagated back via events.
-
-  /**
-   *
-   * @param appContext
-   * @param clientService
-   * @param eventHandler
-   * @param containerSignatureMatcher
-   * @param webUI
-   * @param schedulerDescriptors the list of scheduler descriptors. Tez internal classes will not have the class names populated.
-   *                         An empty list defaults to using the YarnTaskScheduler as the only source.
-   * @param isPureLocalMode whether the AM is running in local mode
-   */
-  @SuppressWarnings("rawtypes")
-  public TaskSchedulerEventHandler(AppContext appContext,
-      DAGClientServer clientService, EventHandler eventHandler, 
-      ContainerSignatureMatcher containerSignatureMatcher, WebUIService webUI,
-      List<NamedEntityDescriptor> schedulerDescriptors, boolean isPureLocalMode) {
-    super(TaskSchedulerEventHandler.class.getName());
-    Preconditions.checkArgument(schedulerDescriptors != null && !schedulerDescriptors.isEmpty(),
-        "TaskSchedulerDescriptors must be specified");
-    this.appContext = appContext;
-    this.eventHandler = eventHandler;
-    this.clientService = clientService;
-    this.containerSignatureMatcher = containerSignatureMatcher;
-    this.webUI = webUI;
-    this.historyUrl = getHistoryUrl();
-    this.isPureLocalMode = isPureLocalMode;
-    this.appCallbackExecutor = createAppCallbackExecutorService();
-    if (this.webUI != null) {
-      this.webUI.setHistoryUrl(this.historyUrl);
-    }
-
-    this.taskSchedulerDescriptors = schedulerDescriptors.toArray(new NamedEntityDescriptor[schedulerDescriptors.size()]);
-
-    taskSchedulers = new TaskScheduler[this.taskSchedulerDescriptors.length];
-    taskSchedulerServiceWrappers = new ServicePluginLifecycleAbstractService[this.taskSchedulerDescriptors.length];
-  }
-
-  public Map<ApplicationAccessType, String> getApplicationAcls() {
-    return appAcls;
-  }
-
-  public void setSignalled(boolean isSignalled) {
-    this.isSignalled = isSignalled;
-    LOG.info("TaskScheduler notified that iSignalled was : " + isSignalled);
-  }
-
-  public int getNumClusterNodes() {
-    return cachedNodeCount;
-  }
-  
-  public Resource getAvailableResources(int schedulerId) {
-    return taskSchedulers[schedulerId].getAvailableResources();
-  }
-
-  public Resource getTotalResources(int schedulerId) {
-    return taskSchedulers[schedulerId].getTotalResources();
-  }
-
-  private ExecutorService createAppCallbackExecutorService() {
-    return Executors.newSingleThreadExecutor(
-        new ThreadFactoryBuilder().setNameFormat("TaskSchedulerAppCallbackExecutor #%d")
-            .setDaemon(true)
-            .build());
-  }
-
-  public synchronized void handleEvent(AMSchedulerEvent sEvent) {
-    LOG.info("Processing the event " + sEvent.toString());
-    switch (sEvent.getType()) {
-    case S_TA_LAUNCH_REQUEST:
-      handleTaLaunchRequest((AMSchedulerEventTALaunchRequest) sEvent);
-      break;
-    case S_TA_ENDED: // TaskAttempt considered complete.
-      AMSchedulerEventTAEnded event = (AMSchedulerEventTAEnded)sEvent;
-      switch(event.getState()) {
-      case FAILED:
-      case KILLED:
-        handleTAUnsuccessfulEnd(event);
-        break;
-      case SUCCEEDED:
-        handleTASucceeded(event);
-        break;
-      default:
-        throw new TezUncheckedException("Unexecpted TA_ENDED state: " + event.getState());
-      }
-      break;
-    case S_CONTAINER_DEALLOCATE:
-      handleContainerDeallocate((AMSchedulerEventDeallocateContainer)sEvent);
-      break;
-    case S_NODE_UNBLACKLISTED:
-      // fall through
-    case S_NODE_BLACKLISTED:
-      handleNodeBlacklistUpdate((AMSchedulerEventNodeBlacklistUpdate)sEvent);
-      break;
-    case S_NODE_UNHEALTHY:
-      break;
-    case S_NODE_HEALTHY:
-      // Consider changing this to work like BLACKLISTING.
-      break;
-    default:
-      break;
-    }
-  }
-
-  @Override
-  public void handle(AMSchedulerEvent event) {
-    int qSize = eventQueue.size();
-    if (qSize != 0 && qSize % 1000 == 0) {
-      LOG.info("Size of event-queue in RMContainerAllocator is " + qSize);
-    }
-    int remCapacity = eventQueue.remainingCapacity();
-    if (remCapacity < 1000) {
-      LOG.warn("Very low remaining capacity in the event-queue "
-          + "of RMContainerAllocator: " + remCapacity);
-    }
-    try {
-      eventQueue.put(event);
-    } catch (InterruptedException e) {
-      throw new TezUncheckedException(e);
-    }
-  }
-
-  @SuppressWarnings("unchecked")
-  private void sendEvent(Event<?> event) {
-    eventHandler.handle(event);
-  }
-
-  private void handleNodeBlacklistUpdate(AMSchedulerEventNodeBlacklistUpdate event) {
-    if (event.getType() == AMSchedulerEventType.S_NODE_BLACKLISTED) {
-      taskSchedulers[event.getSchedulerId()].blacklistNode(event.getNodeId());
-    } else if (event.getType() == AMSchedulerEventType.S_NODE_UNBLACKLISTED) {
-      taskSchedulers[event.getSchedulerId()].unblacklistNode(event.getNodeId());
-    } else {
-      throw new TezUncheckedException("Invalid event type: " + event.getType());
-    }
-  }
-
-  private void handleContainerDeallocate(
-                                  AMSchedulerEventDeallocateContainer event) {
-    ContainerId containerId = event.getContainerId();
-    // TODO what happens to the task that was connected to this container?
-    // current assumption is that it will eventually call handleTaStopRequest
-    //TaskAttempt taskAttempt = (TaskAttempt)
-    taskSchedulers[event.getSchedulerId()].deallocateContainer(containerId);
-    // TODO does this container need to be stopped via C_STOP_REQUEST
-    sendEvent(new AMContainerEventStopRequest(containerId));
-  }
-
-  private void handleTAUnsuccessfulEnd(AMSchedulerEventTAEnded event) {
-    TaskAttempt attempt = event.getAttempt();
-    // Propagate state and failure cause (if any) when informing the scheduler about the de-allocation.
-    boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()]
-        .deallocateTask(attempt, false, event.getTaskAttemptEndReason(), event.getDiagnostics());
-    // use stored value of container id in case the scheduler has removed this
-    // assignment because the task has been deallocated earlier.
-    // retroactive case
-    ContainerId attemptContainerId = attempt.getAssignedContainerID();
-
-    if(!wasContainerAllocated) {
-      LOG.info("Task: " + attempt.getID() +
-          " has no container assignment in the scheduler");
-      if (attemptContainerId != null) {
-        LOG.error("No container allocated to task: " + attempt.getID()
-            + " according to scheduler. Task reported container id: "
-            + attemptContainerId);
-      }
-    }
-
-    if (attemptContainerId != null) {
-      // TODO either ways send the necessary events
-      // Ask the container to stop.
-      sendEvent(new AMContainerEventStopRequest(attemptContainerId));
-      // Inform the Node - the task has asked to be STOPPED / has already
-      // stopped.
-      // AMNodeImpl blacklisting logic does not account for KILLED attempts.
-      sendEvent(new AMNodeEventTaskAttemptEnded(appContext.getAllContainers().
-          get(attemptContainerId).getContainer().getNodeId(), event.getSchedulerId(),
-          attemptContainerId,
-          attempt.getID(), event.getState() == TaskAttemptState.FAILED));
-    }
-  }
-
-  private void handleTASucceeded(AMSchedulerEventTAEnded event) {
-    TaskAttempt attempt = event.getAttempt();
-    ContainerId usedContainerId = event.getUsedContainerId();
-
-    // This could be null if a task fails / is killed before a container is
-    // assigned to it.
-    if (event.getUsedContainerId() != null) {
-      sendEvent(new AMContainerEventTASucceeded(usedContainerId,
-          event.getAttemptID()));
-      sendEvent(new AMNodeEventTaskAttemptSucceeded(appContext.getAllContainers().
-          get(usedContainerId).getContainer().getNodeId(), event.getSchedulerId(), usedContainerId,
-          event.getAttemptID()));
-    }
-
-    boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()].deallocateTask(attempt,
-        true, null, event.getDiagnostics());
-    if (!wasContainerAllocated) {
-      LOG.error("De-allocated successful task: " + attempt.getID()
-          + ", but TaskScheduler reported no container assigned to task");
-    }
-  }
-
-  private void handleTaLaunchRequest(AMSchedulerEventTALaunchRequest event) {
-    TaskAttempt taskAttempt = event.getTaskAttempt();
-    TaskLocationHint locationHint = event.getLocationHint();
-    String hosts[] = null;
-    String racks[] = null;
-    if (locationHint != null) {
-      TaskBasedLocationAffinity taskAffinity = locationHint.getAffinitizedTask();
-      if (taskAffinity != null) {
-        Vertex vertex = appContext.getCurrentDAG().getVertex(taskAffinity.getVertexName());
-        Preconditions.checkNotNull(vertex, "Invalid vertex in task based affinity " + taskAffinity 
-            + " for attempt: " + taskAttempt.getID());
-        int taskIndex = taskAffinity.getTaskIndex(); 
-        Preconditions.checkState(taskIndex >=0 && taskIndex < vertex.getTotalTasks(), 
-            "Invalid taskIndex in task based affinity " + taskAffinity 
-            + " for attempt: " + taskAttempt.getID());
-        TaskAttempt affinityAttempt = vertex.getTask(taskIndex).getSuccessfulAttempt();
-        if (affinityAttempt != null) {
-          Preconditions.checkNotNull(affinityAttempt.getAssignedContainerID(), affinityAttempt.getID());
-          taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt,
-              event.getCapability(),
-              affinityAttempt.getAssignedContainerID(),
-              Priority.newInstance(event.getPriority()),
-              event.getContainerContext(),
-              event);
-          return;
-        }
-        LOG.info("Attempt: " + taskAttempt.getID() + " has task based affinity to " + taskAffinity 
-            + " but no locality information exists for it. Ignoring hint.");
-        // fall through with null hosts/racks
-      } else {
-        hosts = (locationHint.getHosts() != null) ? locationHint
-            .getHosts().toArray(
-                new String[locationHint.getHosts().size()]) : null;
-        racks = (locationHint.getRacks() != null) ? locationHint.getRacks()
-            .toArray(new String[locationHint.getRacks().size()]) : null;
-      }
-    }
-
-    taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt,
-        event.getCapability(),
-        hosts,
-        racks,
-        Priority.newInstance(event.getPriority()),
-        event.getContainerContext(),
-        event);
-  }
-
-  @VisibleForTesting
-  TaskScheduler createTaskScheduler(String host, int port, String trackingUrl,
-                                                   AppContext appContext,
-                                                   NamedEntityDescriptor taskSchedulerDescriptor,
-                                                   long customAppIdIdentifier,
-                                                   int schedulerId) {
-    TaskSchedulerContext rawContext =
-        new TaskSchedulerContextImpl(this, appContext, schedulerId, trackingUrl,
-            customAppIdIdentifier, host, port, taskSchedulerDescriptor.getUserPayload());
-    TaskSchedulerContext wrappedContext = wrapTaskSchedulerContext(rawContext);
-    String schedulerName = taskSchedulerDescriptor.getEntityName();
-    if (schedulerName.equals(TezConstants.getTezYarnServicePluginName())) {
-      return createYarnTaskScheduler(wrappedContext, schedulerId);
-    } else if (schedulerName.equals(TezConstants.getTezUberServicePluginName())) {
-      return createUberTaskScheduler(wrappedContext, schedulerId);
-    } else {
-      return createCustomTaskScheduler(wrappedContext, taskSchedulerDescriptor, schedulerId);
-    }
-  }
-
-  @VisibleForTesting
-  TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
-    return new TaskSchedulerContextImplWrapper(rawContext, appCallbackExecutor);
-  }
-
-  @VisibleForTesting
-  TaskScheduler createYarnTaskScheduler(TaskSchedulerContext taskSchedulerContext,
-                                        int schedulerId) {
-    LOG.info("Creating TaskScheduler: YarnTaskSchedulerService");
-    return new YarnTaskSchedulerService(taskSchedulerContext);
-  }
-
-  @VisibleForTesting
-  TaskScheduler createUberTaskScheduler(TaskSchedulerContext taskSchedulerContext,
-                                        int schedulerId) {
-    LOG.info("Creating TaskScheduler: Local TaskScheduler");
-    return new LocalTaskSchedulerService(taskSchedulerContext);
-  }
-
-  @SuppressWarnings("unchecked")
-  TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext,
-                                          NamedEntityDescriptor taskSchedulerDescriptor,
-                                          int schedulerId) {
-    LOG.info("Creating custom TaskScheduler {}:{}", taskSchedulerDescriptor.getEntityName(),
-        taskSchedulerDescriptor.getClassName());
-    Class<? extends TaskScheduler> taskSchedulerClazz =
-        (Class<? extends TaskScheduler>) ReflectionUtils
-            .getClazz(taskSchedulerDescriptor.getClassName());
-    try {
-      Constructor<? extends TaskScheduler> ctor = taskSchedulerClazz
-          .getConstructor(TaskSchedulerContext.class);
-      return ctor.newInstance(taskSchedulerContext);
-    } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
-      throw new TezUncheckedException(e);
-    }
-  }
-
-  @VisibleForTesting
-  protected void instantiateSchedulers(String host, int port, String trackingUrl,
-                                       AppContext appContext) {
-    // Iterate over the list and create all the taskSchedulers
-    int j = 0;
-    for (int i = 0; i < taskSchedulerDescriptors.length; i++) {
-      long customAppIdIdentifier;
-      if (isPureLocalMode || taskSchedulerDescriptors[i].getEntityName().equals(
-          TezConstants.getTezYarnServicePluginName())) { // Use the app identifier from the appId.
-        customAppIdIdentifier = appContext.getApplicationID().getClusterTimestamp();
-      } else {
-        customAppIdIdentifier = SCHEDULER_APP_ID_BASE + (j++ * SCHEDULER_APP_ID_INCREMENT);
-      }
-      LOG.info("ClusterIdentifier for TaskScheduler [" + i + ":" + taskSchedulerDescriptors[i].getEntityName() + "]=" +
-          customAppIdIdentifier);
-      taskSchedulers[i] = createTaskScheduler(host, port,
-          trackingUrl, appContext, taskSchedulerDescriptors[i], customAppIdIdentifier, i);
-      taskSchedulerServiceWrappers[i] = new ServicePluginLifecycleAbstractService<>(taskSchedulers[i]);
-    }
-  }
-
-  
-  @Override
-  public synchronized void serviceStart() {
-    InetSocketAddress serviceAddr = clientService.getBindAddress();
-    dagAppMaster = appContext.getAppMaster();
-    // if web service is enabled then set tracking url. else disable it (value = "").
-    // the actual url set on the rm web ui will be the proxy url set by WebAppProxyServlet, which
-    // always try to connect to AM and proxy the response. hence it wont work if the webUIService
-    // is not enabled.
-    String trackingUrl = (webUI != null) ? webUI.getTrackingURL() : "";
-    instantiateSchedulers(serviceAddr.getHostName(), serviceAddr.getPort(), trackingUrl, appContext);
-
-    for (int i = 0 ; i < taskSchedulers.length ; i++) {
-      taskSchedulerServiceWrappers[i].init(getConfig());
-      taskSchedulerServiceWrappers[i].start();
-      if (shouldUnregisterFlag.get()) {
-        // Flag may have been set earlier when task scheduler was not initialized
-        // External services could need to talk to some other entity.
-        taskSchedulers[i].setShouldUnregister();
-      }
-    }
-
-    this.eventHandlingThread = new Thread("TaskSchedulerEventHandlerThread") {
-      @Override
-      public void run() {
-
-        AMSchedulerEvent event;
-
-        while (!stopEventHandling && !Thread.currentThread().isInterrupted()) {
-          try {
-            if (TaskSchedulerEventHandler.this.eventQueue.peek() == null) {
-              notifyForTest();
-            }
-            event = TaskSchedulerEventHandler.this.eventQueue.take();
-          } catch (InterruptedException e) {
-            if(!stopEventHandling) {
-              LOG.warn("Continuing after interrupt : ", e);
-            }
-            continue;
-          }
-
-          try {
-            handleEvent(event);
-          } catch (Throwable t) {
-            LOG.error("Error in handling event type " + event.getType()
-                + " to the TaskScheduler", t);
-            // Kill the AM.
-            sendEvent(new DAGAppMasterEvent(DAGAppMasterEventType.INTERNAL_ERROR));
-            return;
-          } finally {
-            notifyForTest();
-          }
-        }
-      }
-    };
-    this.eventHandlingThread.start();
-  }
-  
-  protected void notifyForTest() {
-  }
-
-  public void initiateStop() {
-    for (int i = 0 ; i < taskSchedulers.length ; i++) {
-      taskSchedulers[i].initiateStop();
-    }
-  }
-
-  @Override
-  public void serviceStop() throws InterruptedException {
-    synchronized(this) {
-      this.stopEventHandling = true;
-      if (eventHandlingThread != null)
-        eventHandlingThread.interrupt();
-    }
-    for (int i = 0 ; i < taskSchedulers.length ; i++) {
-      if (taskSchedulers[i] != null) {
-        taskSchedulerServiceWrappers[i].stop();
-      }
-    }
-    LOG.info("Shutting down AppCallbackExecutor");
-    appCallbackExecutor.shutdownNow();
-    appCallbackExecutor.awaitTermination(1000l, TimeUnit.MILLISECONDS);
-  }
-
-  // TaskSchedulerAppCallback methods with schedulerId, where relevant
-  public synchronized void taskAllocated(int schedulerId, Object task,
-                                           Object appCookie,
-                                           Container container) {
-    AMSchedulerEventTALaunchRequest event =
-        (AMSchedulerEventTALaunchRequest) appCookie;
-    ContainerId containerId = container.getId();
-    if (appContext.getAllContainers()
-        .addContainerIfNew(container, schedulerId, event.getLauncherId(),
-            event.getTaskCommId())) {
-      appContext.getNodeTracker().nodeSeen(container.getNodeId(), schedulerId);
-      sendEvent(new AMNodeEventContainerAllocated(container
-          .getNodeId(), schedulerId, container.getId()));
-    }
-
-
-    TaskAttempt taskAttempt = event.getTaskAttempt();
-    // TODO - perhaps check if the task still needs this container
-    // because the deallocateTask downcall may have raced with the
-    // taskAllocated() upcall
-    assert task.equals(taskAttempt);
- 
-    if (appContext.getAllContainers().get(containerId).getState() == AMContainerState.ALLOCATED) {
-      sendEvent(new AMContainerEventLaunchRequest(containerId, taskAttempt.getVertexID(),
-          event.getContainerContext(), event.getLauncherId(), event.getTaskCommId()));
-    }
-    sendEvent(new DAGEventSchedulerUpdateTAAssigned(taskAttempt, container));
-    sendEvent(new AMContainerEventAssignTA(containerId, taskAttempt.getID(),
-        event.getRemoteTaskSpec(), event.getContainerContext().getLocalResources(), event
-            .getContainerContext().getCredentials(), event.getPriority()));
-  }
-
-  public synchronized void containerCompleted(int schedulerId, Object task, ContainerStatus containerStatus) {
-    // SchedulerId isn't used here since no node updates are sent out
-    // Inform the Containers about completion.
-    AMContainer amContainer = appContext.getAllContainers().get(containerStatus.getContainerId());
-    if (amContainer != null) {
-      String message = "Container completed. ";
-      TaskAttemptTerminationCause errCause = TaskAttemptTerminationCause.CONTAINER_EXITED;
-      int exitStatus = containerStatus.getExitStatus();
-      if (exitStatus == ContainerExitStatus.PREEMPTED) {
-        message = "Container preempted externally. ";
-        errCause = TaskAttemptTerminationCause.EXTERNAL_PREEMPTION;
-      } else if (exitStatus == ContainerExitStatus.DISKS_FAILED) {
-        message = "Container disk failed. ";
-        errCause = TaskAttemptTerminationCause.NODE_DISK_ERROR;
-      } else if (exitStatus != ContainerExitStatus.SUCCESS){
-        message = "Container failed, exitCode=" + exitStatus + ". ";
-      }
-      if (containerStatus.getDiagnostics() != null) {
-        message += containerStatus.getDiagnostics();
-      }
-      sendEvent(new AMContainerEventCompleted(amContainer.getContainerId(), exitStatus, message, errCause));
-    }
-  }
-
-  public synchronized void containerBeingReleased(int schedulerId, ContainerId containerId) {
-    // SchedulerId isn't used here since no node updates are sent out
-    AMContainer amContainer = appContext.getAllContainers().get(containerId);
-    if (amContainer != null) {
-      sendEvent(new AMContainerEventStopRequest(containerId));
-    }
-  }
-
-  @SuppressWarnings("unchecked")
-  public synchronized void nodesUpdated(int schedulerId, List<NodeReport> updatedNodes) {
-    for (NodeReport nr : updatedNodes) {
-      // Scheduler will find out from the node, if at all.
-      // Relying on the RM to not allocate containers on an unhealthy node.
-      eventHandler.handle(new AMNodeEventStateChanged(nr, schedulerId));
-    }
-  }
-
-  public synchronized void appShutdownRequested(int schedulerId) {
-    // This can happen if the RM has been restarted. If it is in that state,
-    // this application must clean itself up.
-    LOG.info("App shutdown requested by scheduler {}", schedulerId);
-    sendEvent(new DAGAppMasterEvent(DAGAppMasterEventType.AM_REBOOT));
-  }
-
-  public synchronized void setApplicationRegistrationData(
-      int schedulerId,
-      Resource maxContainerCapability,
-      Map<ApplicationAccessType, String> appAcls, 
-      ByteBuffer clientAMSecretKey) {
-    this.appContext.getClusterInfo().setMaxContainerCapability(
-        maxContainerCapability);
-    this.appAcls = appAcls;
-    this.clientService.setClientAMSecretKey(clientAMSecretKey);
-  }
-
-  // Not synchronized to avoid deadlocks from TaskScheduler callbacks.
-  // TaskScheduler uses a separate thread for it's callbacks. Since this method
-  // returns a value which is required, the TaskScheduler wait for the call to
-  // complete and can hence lead to a deadlock if called from within a TSEH lock.
-  public AppFinalStatus getFinalAppStatus() {
-    FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
-    StringBuffer sb = new StringBuffer();
-    if (dagAppMaster == null) {
-      finishState = FinalApplicationStatus.UNDEFINED;
-      sb.append("App not yet initialized");
-    } else {
-      DAGAppMasterState appMasterState = dagAppMaster.getState();
-      if (appMasterState == DAGAppMasterState.SUCCEEDED) {
-        finishState = FinalApplicationStatus.SUCCEEDED;
-      } else if (appMasterState == DAGAppMasterState.KILLED
-          || (appMasterState == DAGAppMasterState.RUNNING && isSignalled)) {
-        finishState = FinalApplicationStatus.KILLED;
-      } else if (appMasterState == DAGAppMasterState.FAILED
-          || appMasterState == DAGAppMasterState.ERROR) {
-        finishState = FinalApplicationStatus.FAILED;
-      } else {
-        finishState = FinalApplicationStatus.UNDEFINED;
-      }
-      List<String> diagnostics = dagAppMaster.getDiagnostics();
-      if(diagnostics != null) {
-        for (String s : diagnostics) {
-          sb.append(s).append("\n");
-        }
-      }
-    }
-    if(LOG.isDebugEnabled()) {
-      LOG.debug("Setting job diagnostics to " + sb.toString());
-    }
-
-    // if history url is set use the same, if historyUrl is set to "" then rm ui disables the
-    // history url
-    return new AppFinalStatus(finishState, sb.toString(), historyUrl);
-  }
-
-
-
-  // Not synchronized to avoid deadlocks from TaskScheduler callbacks.
-  // TaskScheduler uses a separate thread for it's callbacks. Since this method
-  // returns a value which is required, the TaskScheduler wait for the call to
-  // complete and can hence lead to a deadlock if called from within a TSEH lock.
-  public float getProgress(int schedulerId) {
-    // at this point allocate has been called and so node count must be available
-    // may change after YARN-1722
-    // This is a heartbeat in from the scheduler into the APP, and is being used to piggy-back and
-    // node updates from the cluster.
-
-    // Doubles as a mechanism to update node counts periodically. Hence schedulerId required.
-
-    // TODO Handle this in TEZ-2124. Need a way to know which scheduler is calling in.
-    int nodeCount = taskSchedulers[0].getClusterNodeCount();
-    if (nodeCount != cachedNodeCount) {
-      cachedNodeCount = nodeCount;
-      sendEvent(new AMNodeEventNodeCountUpdated(cachedNodeCount, schedulerId));
-    }
-    return dagAppMaster.getProgress();
-  }
-
-  public void onError(int schedulerId, Throwable t) {
-    LOG.info("Error reported by scheduler {} - {}", schedulerId, t);
-    sendEvent(new DAGAppMasterEventSchedulingServiceError(t));
-  }
-
-  public void dagCompleted() {
-    for (int i = 0 ; i < taskSchedulers.length ; i++) {
-      taskSchedulers[i].dagComplete();
-    }
-  }
-
-  public void dagSubmitted() {
-    // Nothing to do right now. Indicates that a new DAG has been submitted and
-    // the context has updated information.
-  }
-
-  public void preemptContainer(int schedulerId, ContainerId containerId) {
-    // TODO Why is this making a call back into the scheduler, when the call is originating from there.
-    // An AMContainer instance should already exist if an attempt is being made to preempt it
-    AMContainer amContainer = appContext.getAllContainers().get(containerId);
-    taskSchedulers[amContainer.getTaskSchedulerIdentifier()].deallocateContainer(containerId);
-    // Inform the Containers about completion.
-    sendEvent(new AMContainerEventCompleted(containerId, ContainerExitStatus.INVALID,
-        "Container preempted internally", TaskAttemptTerminationCause.INTERNAL_PREEMPTION));
-  }
-
-  public void setShouldUnregisterFlag() {
-    LOG.info("TaskScheduler notified that it should unregister from RM");
-    this.shouldUnregisterFlag.set(true);
-    for (int i = 0 ; i < taskSchedulers.length ; i++) {
-      if (this.taskSchedulers[i] != null) {
-        this.taskSchedulers[i].setShouldUnregister();
-      }
-    }
-  }
-
-  public ContainerSignatureMatcher getContainerSignatureMatcher() {
-    return containerSignatureMatcher;
-  }
-
-  public boolean hasUnregistered() {
-    boolean result = true;
-    for (int i = 0 ; i < taskSchedulers.length ; i++) {
-      result = result & this.taskSchedulers[i].hasUnregistered();
-      if (result == false) {
-        return result;
-      }
-    }
-    return result;
-  }
-
-  @VisibleForTesting
-  public String getHistoryUrl() {
-    Configuration config = this.appContext.getAMConf();
-    String historyUrl = "";
-
-    String loggingClass =  config.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "");
-    String historyUrlTemplate = config.get(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
-            TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE_DEFAULT);
-    String historyUrlBase = config.get(TezConfiguration.TEZ_HISTORY_URL_BASE, "");
-
-
-    if (loggingClass.equals("org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService") &&
-        !historyUrlTemplate.isEmpty() &&
-        !historyUrlBase.isEmpty()) {
-      // replace the placeholders, while tolerating extra or missing "/" in input. replace all
-      // instances of consecutive "/" with single (except for the http(s):// case
-      historyUrl = historyUrlTemplate
-          .replaceAll(APPLICATION_ID_PLACEHOLDER, appContext.getApplicationID().toString())
-          .replaceAll(HISTORY_URL_BASE, historyUrlBase)
-          .replaceAll("([^:])/{2,}", "$1/");
-
-      // make sure we have a valid scheme
-      if (!historyUrl.startsWith("http")) {
-        historyUrl = "http://" + historyUrl;
-      }
-    }
-
-    return historyUrl;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
new file mode 100644
index 0000000..29143a2
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
@@ -0,0 +1,786 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
+import org.apache.tez.serviceplugins.api.TaskScheduler;
+import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
+import org.apache.tez.serviceplugins.api.TaskSchedulerContext.AppFinalStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.TaskLocationHint;
+import org.apache.tez.dag.api.TaskLocationHint.TaskBasedLocationAffinity;
+import org.apache.tez.dag.api.client.DAGClientServer;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.DAGAppMaster;
+import org.apache.tez.dag.app.DAGAppMasterState;
+import org.apache.tez.dag.app.dag.TaskAttempt;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.event.DAGAppMasterEvent;
+import org.apache.tez.dag.app.dag.event.DAGAppMasterEventSchedulingServiceError;
+import org.apache.tez.dag.app.dag.event.DAGAppMasterEventType;
+import org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdateTAAssigned;
+import org.apache.tez.dag.app.rm.container.AMContainer;
+import org.apache.tez.dag.app.rm.container.AMContainerEventAssignTA;
+import org.apache.tez.dag.app.rm.container.AMContainerEventCompleted;
+import org.apache.tez.dag.app.rm.container.AMContainerEventLaunchRequest;
+import org.apache.tez.dag.app.rm.container.AMContainerEventStopRequest;
+import org.apache.tez.dag.app.rm.container.AMContainerEventTASucceeded;
+import org.apache.tez.dag.app.rm.container.AMContainerState;
+import org.apache.tez.common.ContainerSignatureMatcher;
+import org.apache.tez.dag.app.rm.node.AMNodeEventContainerAllocated;
+import org.apache.tez.dag.app.rm.node.AMNodeEventNodeCountUpdated;
+import org.apache.tez.dag.app.rm.node.AMNodeEventStateChanged;
+import org.apache.tez.dag.app.rm.node.AMNodeEventTaskAttemptEnded;
+import org.apache.tez.dag.app.rm.node.AMNodeEventTaskAttemptSucceeded;
+import org.apache.tez.dag.app.web.WebUIService;
+import org.apache.tez.dag.records.TaskAttemptTerminationCause;
+
+import com.google.common.base.Preconditions;
+
+
+public class TaskSchedulerManager extends AbstractService implements
+                                               EventHandler<AMSchedulerEvent> {
+  static final Logger LOG = LoggerFactory.getLogger(TaskSchedulerManager.class);
+
+  static final String APPLICATION_ID_PLACEHOLDER = "__APPLICATION_ID__";
+  static final String HISTORY_URL_BASE = "__HISTORY_URL_BASE__";
+
+  protected final AppContext appContext;
+  @SuppressWarnings("rawtypes")
+  private final EventHandler eventHandler;
+  private final String historyUrl;
+  private DAGAppMaster dagAppMaster;
+  private Map<ApplicationAccessType, String> appAcls = null;
+  private Thread eventHandlingThread;
+  private volatile boolean stopEventHandling;
+  // Has a signal (SIGTERM etc) been issued?
+  protected volatile boolean isSignalled = false;
+  final DAGClientServer clientService;
+  private final ContainerSignatureMatcher containerSignatureMatcher;
+  private int cachedNodeCount = -1;
+  private AtomicBoolean shouldUnregisterFlag =
+      new AtomicBoolean(false);
+  private final WebUIService webUI;
+  private final NamedEntityDescriptor[] taskSchedulerDescriptors;
+  protected final TaskScheduler[]taskSchedulers;
+  protected final ServicePluginLifecycleAbstractService []taskSchedulerServiceWrappers;
+
+  // Single executor service shared by all Schedulers for context callbacks
+  @VisibleForTesting
+  final ExecutorService appCallbackExecutor;
+
+  private final boolean isPureLocalMode;
+  // If running in non local-only mode, the YARN task scheduler will always run to take care of
+  // registration with YARN and heartbeats to YARN.
+  // Splitting registration and heartbeats is not straight-forward due to the taskScheduler being
+  // tied to a ContainerRequestType.
+  // Custom AppIds to avoid container conflicts if there's multiple sources
+  private final long SCHEDULER_APP_ID_BASE = 111101111;
+  private final long SCHEDULER_APP_ID_INCREMENT = 111111111;
+
+  BlockingQueue<AMSchedulerEvent> eventQueue
+                              = new LinkedBlockingQueue<AMSchedulerEvent>();
+
+  // Not tracking container / task to schedulerId. Instead relying on everything flowing through
+  // the system and being propagated back via events.
+
+  /**
+   *
+   * @param appContext
+   * @param clientService
+   * @param eventHandler
+   * @param containerSignatureMatcher
+   * @param webUI
+   * @param schedulerDescriptors the list of scheduler descriptors. Tez internal classes will not have the class names populated.
+   *                         An empty list defaults to using the YarnTaskScheduler as the only source.
+   * @param isPureLocalMode whether the AM is running in local mode
+   */
+  @SuppressWarnings("rawtypes")
+  public TaskSchedulerManager(AppContext appContext,
+                              DAGClientServer clientService, EventHandler eventHandler,
+                              ContainerSignatureMatcher containerSignatureMatcher,
+                              WebUIService webUI,
+                              List<NamedEntityDescriptor> schedulerDescriptors,
+                              boolean isPureLocalMode) {
+    super(TaskSchedulerManager.class.getName());
+    Preconditions.checkArgument(schedulerDescriptors != null && !schedulerDescriptors.isEmpty(),
+        "TaskSchedulerDescriptors must be specified");
+    this.appContext = appContext;
+    this.eventHandler = eventHandler;
+    this.clientService = clientService;
+    this.containerSignatureMatcher = containerSignatureMatcher;
+    this.webUI = webUI;
+    this.historyUrl = getHistoryUrl();
+    this.isPureLocalMode = isPureLocalMode;
+    this.appCallbackExecutor = createAppCallbackExecutorService();
+    if (this.webUI != null) {
+      this.webUI.setHistoryUrl(this.historyUrl);
+    }
+
+    this.taskSchedulerDescriptors = schedulerDescriptors.toArray(new NamedEntityDescriptor[schedulerDescriptors.size()]);
+
+    taskSchedulers = new TaskScheduler[this.taskSchedulerDescriptors.length];
+    taskSchedulerServiceWrappers = new ServicePluginLifecycleAbstractService[this.taskSchedulerDescriptors.length];
+  }
+
+  public Map<ApplicationAccessType, String> getApplicationAcls() {
+    return appAcls;
+  }
+
+  public void setSignalled(boolean isSignalled) {
+    this.isSignalled = isSignalled;
+    LOG.info("TaskScheduler notified that iSignalled was : " + isSignalled);
+  }
+
+  public int getNumClusterNodes() {
+    return cachedNodeCount;
+  }
+  
+  public Resource getAvailableResources(int schedulerId) {
+    return taskSchedulers[schedulerId].getAvailableResources();
+  }
+
+  public Resource getTotalResources(int schedulerId) {
+    return taskSchedulers[schedulerId].getTotalResources();
+  }
+
+  private ExecutorService createAppCallbackExecutorService() {
+    return Executors.newSingleThreadExecutor(
+        new ThreadFactoryBuilder().setNameFormat("TaskSchedulerAppCallbackExecutor #%d")
+            .setDaemon(true)
+            .build());
+  }
+
+  public synchronized void handleEvent(AMSchedulerEvent sEvent) {
+    LOG.info("Processing the event " + sEvent.toString());
+    switch (sEvent.getType()) {
+    case S_TA_LAUNCH_REQUEST:
+      handleTaLaunchRequest((AMSchedulerEventTALaunchRequest) sEvent);
+      break;
+    case S_TA_ENDED: // TaskAttempt considered complete.
+      AMSchedulerEventTAEnded event = (AMSchedulerEventTAEnded)sEvent;
+      switch(event.getState()) {
+      case FAILED:
+      case KILLED:
+        handleTAUnsuccessfulEnd(event);
+        break;
+      case SUCCEEDED:
+        handleTASucceeded(event);
+        break;
+      default:
+        throw new TezUncheckedException("Unexecpted TA_ENDED state: " + event.getState());
+      }
+      break;
+    case S_CONTAINER_DEALLOCATE:
+      handleContainerDeallocate((AMSchedulerEventDeallocateContainer)sEvent);
+      break;
+    case S_NODE_UNBLACKLISTED:
+      // fall through
+    case S_NODE_BLACKLISTED:
+      handleNodeBlacklistUpdate((AMSchedulerEventNodeBlacklistUpdate)sEvent);
+      break;
+    case S_NODE_UNHEALTHY:
+      break;
+    case S_NODE_HEALTHY:
+      // Consider changing this to work like BLACKLISTING.
+      break;
+    default:
+      break;
+    }
+  }
+
+  @Override
+  public void handle(AMSchedulerEvent event) {
+    int qSize = eventQueue.size();
+    if (qSize != 0 && qSize % 1000 == 0) {
+      LOG.info("Size of event-queue in RMContainerAllocator is " + qSize);
+    }
+    int remCapacity = eventQueue.remainingCapacity();
+    if (remCapacity < 1000) {
+      LOG.warn("Very low remaining capacity in the event-queue "
+          + "of RMContainerAllocator: " + remCapacity);
+    }
+    try {
+      eventQueue.put(event);
+    } catch (InterruptedException e) {
+      throw new TezUncheckedException(e);
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  private void sendEvent(Event<?> event) {
+    eventHandler.handle(event);
+  }
+
+  private void handleNodeBlacklistUpdate(AMSchedulerEventNodeBlacklistUpdate event) {
+    if (event.getType() == AMSchedulerEventType.S_NODE_BLACKLISTED) {
+      taskSchedulers[event.getSchedulerId()].blacklistNode(event.getNodeId());
+    } else if (event.getType() == AMSchedulerEventType.S_NODE_UNBLACKLISTED) {
+      taskSchedulers[event.getSchedulerId()].unblacklistNode(event.getNodeId());
+    } else {
+      throw new TezUncheckedException("Invalid event type: " + event.getType());
+    }
+  }
+
+  private void handleContainerDeallocate(
+                                  AMSchedulerEventDeallocateContainer event) {
+    ContainerId containerId = event.getContainerId();
+    // TODO what happens to the task that was connected to this container?
+    // current assumption is that it will eventually call handleTaStopRequest
+    //TaskAttempt taskAttempt = (TaskAttempt)
+    taskSchedulers[event.getSchedulerId()].deallocateContainer(containerId);
+    // TODO does this container need to be stopped via C_STOP_REQUEST
+    sendEvent(new AMContainerEventStopRequest(containerId));
+  }
+
+  private void handleTAUnsuccessfulEnd(AMSchedulerEventTAEnded event) {
+    TaskAttempt attempt = event.getAttempt();
+    // Propagate state and failure cause (if any) when informing the scheduler about the de-allocation.
+    boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()]
+        .deallocateTask(attempt, false, event.getTaskAttemptEndReason(), event.getDiagnostics());
+    // use stored value of container id in case the scheduler has removed this
+    // assignment because the task has been deallocated earlier.
+    // retroactive case
+    ContainerId attemptContainerId = attempt.getAssignedContainerID();
+
+    if(!wasContainerAllocated) {
+      LOG.info("Task: " + attempt.getID() +
+          " has no container assignment in the scheduler");
+      if (attemptContainerId != null) {
+        LOG.error("No container allocated to task: " + attempt.getID()
+            + " according to scheduler. Task reported container id: "
+            + attemptContainerId);
+      }
+    }
+
+    if (attemptContainerId != null) {
+      // TODO either ways send the necessary events
+      // Ask the container to stop.
+      sendEvent(new AMContainerEventStopRequest(attemptContainerId));
+      // Inform the Node - the task has asked to be STOPPED / has already
+      // stopped.
+      // AMNodeImpl blacklisting logic does not account for KILLED attempts.
+      sendEvent(new AMNodeEventTaskAttemptEnded(appContext.getAllContainers().
+          get(attemptContainerId).getContainer().getNodeId(), event.getSchedulerId(),
+          attemptContainerId,
+          attempt.getID(), event.getState() == TaskAttemptState.FAILED));
+    }
+  }
+
+  private void handleTASucceeded(AMSchedulerEventTAEnded event) {
+    TaskAttempt attempt = event.getAttempt();
+    ContainerId usedContainerId = event.getUsedContainerId();
+
+    // This could be null if a task fails / is killed before a container is
+    // assigned to it.
+    if (event.getUsedContainerId() != null) {
+      sendEvent(new AMContainerEventTASucceeded(usedContainerId,
+          event.getAttemptID()));
+      sendEvent(new AMNodeEventTaskAttemptSucceeded(appContext.getAllContainers().
+          get(usedContainerId).getContainer().getNodeId(), event.getSchedulerId(), usedContainerId,
+          event.getAttemptID()));
+    }
+
+    boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()].deallocateTask(attempt,
+        true, null, event.getDiagnostics());
+    if (!wasContainerAllocated) {
+      LOG.error("De-allocated successful task: " + attempt.getID()
+          + ", but TaskScheduler reported no container assigned to task");
+    }
+  }
+
+  private void handleTaLaunchRequest(AMSchedulerEventTALaunchRequest event) {
+    TaskAttempt taskAttempt = event.getTaskAttempt();
+    TaskLocationHint locationHint = event.getLocationHint();
+    String hosts[] = null;
+    String racks[] = null;
+    if (locationHint != null) {
+      TaskBasedLocationAffinity taskAffinity = locationHint.getAffinitizedTask();
+      if (taskAffinity != null) {
+        Vertex vertex = appContext.getCurrentDAG().getVertex(taskAffinity.getVertexName());
+        Preconditions.checkNotNull(vertex, "Invalid vertex in task based affinity " + taskAffinity 
+            + " for attempt: " + taskAttempt.getID());
+        int taskIndex = taskAffinity.getTaskIndex(); 
+        Preconditions.checkState(taskIndex >=0 && taskIndex < vertex.getTotalTasks(), 
+            "Invalid taskIndex in task based affinity " + taskAffinity 
+            + " for attempt: " + taskAttempt.getID());
+        TaskAttempt affinityAttempt = vertex.getTask(taskIndex).getSuccessfulAttempt();
+        if (affinityAttempt != null) {
+          Preconditions.checkNotNull(affinityAttempt.getAssignedContainerID(), affinityAttempt.getID());
+          taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt,
+              event.getCapability(),
+              affinityAttempt.getAssignedContainerID(),
+              Priority.newInstance(event.getPriority()),
+              event.getContainerContext(),
+              event);
+          return;
+        }
+        LOG.info("Attempt: " + taskAttempt.getID() + " has task based affinity to " + taskAffinity 
+            + " but no locality information exists for it. Ignoring hint.");
+        // fall through with null hosts/racks
+      } else {
+        hosts = (locationHint.getHosts() != null) ? locationHint
+            .getHosts().toArray(
+                new String[locationHint.getHosts().size()]) : null;
+        racks = (locationHint.getRacks() != null) ? locationHint.getRacks()
+            .toArray(new String[locationHint.getRacks().size()]) : null;
+      }
+    }
+
+    taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt,
+        event.getCapability(),
+        hosts,
+        racks,
+        Priority.newInstance(event.getPriority()),
+        event.getContainerContext(),
+        event);
+  }
+
+  @VisibleForTesting
+  TaskScheduler createTaskScheduler(String host, int port, String trackingUrl,
+                                                   AppContext appContext,
+                                                   NamedEntityDescriptor taskSchedulerDescriptor,
+                                                   long customAppIdIdentifier,
+                                                   int schedulerId) {
+    TaskSchedulerContext rawContext =
+        new TaskSchedulerContextImpl(this, appContext, schedulerId, trackingUrl,
+            customAppIdIdentifier, host, port, taskSchedulerDescriptor.getUserPayload());
+    TaskSchedulerContext wrappedContext = wrapTaskSchedulerContext(rawContext);
+    String schedulerName = taskSchedulerDescriptor.getEntityName();
+    if (schedulerName.equals(TezConstants.getTezYarnServicePluginName())) {
+      return createYarnTaskScheduler(wrappedContext, schedulerId);
+    } else if (schedulerName.equals(TezConstants.getTezUberServicePluginName())) {
+      return createUberTaskScheduler(wrappedContext, schedulerId);
+    } else {
+      return createCustomTaskScheduler(wrappedContext, taskSchedulerDescriptor, schedulerId);
+    }
+  }
+
+  @VisibleForTesting
+  TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
+    return new TaskSchedulerContextImplWrapper(rawContext, appCallbackExecutor);
+  }
+
+  @VisibleForTesting
+  TaskScheduler createYarnTaskScheduler(TaskSchedulerContext taskSchedulerContext,
+                                        int schedulerId) {
+    LOG.info("Creating TaskScheduler: YarnTaskSchedulerService");
+    return new YarnTaskSchedulerService(taskSchedulerContext);
+  }
+
+  @VisibleForTesting
+  TaskScheduler createUberTaskScheduler(TaskSchedulerContext taskSchedulerContext,
+                                        int schedulerId) {
+    LOG.info("Creating TaskScheduler: Local TaskScheduler");
+    return new LocalTaskSchedulerService(taskSchedulerContext);
+  }
+
+  @SuppressWarnings("unchecked")
+  TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext,
+                                          NamedEntityDescriptor taskSchedulerDescriptor,
+                                          int schedulerId) {
+    LOG.info("Creating custom TaskScheduler {}:{}", taskSchedulerDescriptor.getEntityName(),
+        taskSchedulerDescriptor.getClassName());
+    Class<? extends TaskScheduler> taskSchedulerClazz =
+        (Class<? extends TaskScheduler>) ReflectionUtils
+            .getClazz(taskSchedulerDescriptor.getClassName());
+    try {
+      Constructor<? extends TaskScheduler> ctor = taskSchedulerClazz
+          .getConstructor(TaskSchedulerContext.class);
+      return ctor.newInstance(taskSchedulerContext);
+    } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
+      throw new TezUncheckedException(e);
+    }
+  }
+
+  @VisibleForTesting
+  protected void instantiateSchedulers(String host, int port, String trackingUrl,
+                                       AppContext appContext) {
+    // Iterate over the list and create all the taskSchedulers
+    int j = 0;
+    for (int i = 0; i < taskSchedulerDescriptors.length; i++) {
+      long customAppIdIdentifier;
+      if (isPureLocalMode || taskSchedulerDescriptors[i].getEntityName().equals(
+          TezConstants.getTezYarnServicePluginName())) { // Use the app identifier from the appId.
+        customAppIdIdentifier = appContext.getApplicationID().getClusterTimestamp();
+      } else {
+        customAppIdIdentifier = SCHEDULER_APP_ID_BASE + (j++ * SCHEDULER_APP_ID_INCREMENT);
+      }
+      LOG.info("ClusterIdentifier for TaskScheduler [" + i + ":" + taskSchedulerDescriptors[i].getEntityName() + "]=" +
+          customAppIdIdentifier);
+      taskSchedulers[i] = createTaskScheduler(host, port,
+          trackingUrl, appContext, taskSchedulerDescriptors[i], customAppIdIdentifier, i);
+      taskSchedulerServiceWrappers[i] = new ServicePluginLifecycleAbstractService<>(taskSchedulers[i]);
+    }
+  }
+
+  
+  @Override
+  public synchronized void serviceStart() {
+    InetSocketAddress serviceAddr = clientService.getBindAddress();
+    dagAppMaster = appContext.getAppMaster();
+    // if web service is enabled then set tracking url. else disable it (value = "").
+    // the actual url set on the rm web ui will be the proxy url set by WebAppProxyServlet, which
+    // always try to connect to AM and proxy the response. hence it wont work if the webUIService
+    // is not enabled.
+    String trackingUrl = (webUI != null) ? webUI.getTrackingURL() : "";
+    instantiateSchedulers(serviceAddr.getHostName(), serviceAddr.getPort(), trackingUrl, appContext);
+
+    for (int i = 0 ; i < taskSchedulers.length ; i++) {
+      taskSchedulerServiceWrappers[i].init(getConfig());
+      taskSchedulerServiceWrappers[i].start();
+      if (shouldUnregisterFlag.get()) {
+        // Flag may have been set earlier when task scheduler was not initialized
+        // External services could need to talk to some other entity.
+        taskSchedulers[i].setShouldUnregister();
+      }
+    }
+
+    this.eventHandlingThread = new Thread("TaskSchedulerEventHandlerThread") {
+      @Override
+      public void run() {
+
+        AMSchedulerEvent event;
+
+        while (!stopEventHandling && !Thread.currentThread().isInterrupted()) {
+          try {
+            if (TaskSchedulerManager.this.eventQueue.peek() == null) {
+              notifyForTest();
+            }
+            event = TaskSchedulerManager.this.eventQueue.take();
+          } catch (InterruptedException e) {
+            if(!stopEventHandling) {
+              LOG.warn("Continuing after interrupt : ", e);
+            }
+            continue;
+          }
+
+          try {
+            handleEvent(event);
+          } catch (Throwable t) {
+            LOG.error("Error in handling event type " + event.getType()
+                + " to the TaskScheduler", t);
+            // Kill the AM.
+            sendEvent(new DAGAppMasterEvent(DAGAppMasterEventType.INTERNAL_ERROR));
+            return;
+          } finally {
+            notifyForTest();
+          }
+        }
+      }
+    };
+    this.eventHandlingThread.start();
+  }
+  
+  protected void notifyForTest() {
+  }
+
+  public void initiateStop() {
+    for (int i = 0 ; i < taskSchedulers.length ; i++) {
+      taskSchedulers[i].initiateStop();
+    }
+  }
+
+  @Override
+  public void serviceStop() throws InterruptedException {
+    synchronized(this) {
+      this.stopEventHandling = true;
+      if (eventHandlingThread != null)
+        eventHandlingThread.interrupt();
+    }
+    for (int i = 0 ; i < taskSchedulers.length ; i++) {
+      if (taskSchedulers[i] != null) {
+        taskSchedulerServiceWrappers[i].stop();
+      }
+    }
+    LOG.info("Shutting down AppCallbackExecutor");
+    appCallbackExecutor.shutdownNow();
+    appCallbackExecutor.awaitTermination(1000l, TimeUnit.MILLISECONDS);
+  }
+
+  // TaskSchedulerAppCallback methods with schedulerId, where relevant
+  public synchronized void taskAllocated(int schedulerId, Object task,
+                                           Object appCookie,
+                                           Container container) {
+    AMSchedulerEventTALaunchRequest event =
+        (AMSchedulerEventTALaunchRequest) appCookie;
+    ContainerId containerId = container.getId();
+    if (appContext.getAllContainers()
+        .addContainerIfNew(container, schedulerId, event.getLauncherId(),
+            event.getTaskCommId())) {
+      appContext.getNodeTracker().nodeSeen(container.getNodeId(), schedulerId);
+      sendEvent(new AMNodeEventContainerAllocated(container
+          .getNodeId(), schedulerId, container.getId()));
+    }
+
+
+    TaskAttempt taskAttempt = event.getTaskAttempt();
+    // TODO - perhaps check if the task still needs this container
+    // because the deallocateTask downcall may have raced with the
+    // taskAllocated() upcall
+    assert task.equals(taskAttempt);
+ 
+    if (appContext.getAllContainers().get(containerId).getState() == AMContainerState.ALLOCATED) {
+      sendEvent(new AMContainerEventLaunchRequest(containerId, taskAttempt.getVertexID(),
+          event.getContainerContext(), event.getLauncherId(), event.getTaskCommId()));
+    }
+    sendEvent(new DAGEventSchedulerUpdateTAAssigned(taskAttempt, container));
+    sendEvent(new AMContainerEventAssignTA(containerId, taskAttempt.getID(),
+        event.getRemoteTaskSpec(), event.getContainerContext().getLocalResources(), event
+            .getContainerContext().getCredentials(), event.getPriority()));
+  }
+
+  public synchronized void containerCompleted(int schedulerId, Object task, ContainerStatus containerStatus) {
+    // SchedulerId isn't used here since no node updates are sent out
+    // Inform the Containers about completion.
+    AMContainer amContainer = appContext.getAllContainers().get(containerStatus.getContainerId());
+    if (amContainer != null) {
+      String message = "Container completed. ";
+      TaskAttemptTerminationCause errCause = TaskAttemptTerminationCause.CONTAINER_EXITED;
+      int exitStatus = containerStatus.getExitStatus();
+      if (exitStatus == ContainerExitStatus.PREEMPTED) {
+        message = "Container preempted externally. ";
+        errCause = TaskAttemptTerminationCause.EXTERNAL_PREEMPTION;
+      } else if (exitStatus == ContainerExitStatus.DISKS_FAILED) {
+        message = "Container disk failed. ";
+        errCause = TaskAttemptTerminationCause.NODE_DISK_ERROR;
+      } else if (exitStatus != ContainerExitStatus.SUCCESS){
+        message = "Container failed, exitCode=" + exitStatus + ". ";
+      }
+      if (containerStatus.getDiagnostics() != null) {
+        message += containerStatus.getDiagnostics();
+      }
+      sendEvent(new AMContainerEventCompleted(amContainer.getContainerId(), exitStatus, message, errCause));
+    }
+  }
+
+  public synchronized void containerBeingReleased(int schedulerId, ContainerId containerId) {
+    // SchedulerId isn't used here since no node updates are sent out
+    AMContainer amContainer = appContext.getAllContainers().get(containerId);
+    if (amContainer != null) {
+      sendEvent(new AMContainerEventStopRequest(containerId));
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  public synchronized void nodesUpdated(int schedulerId, List<NodeReport> updatedNodes) {
+    for (NodeReport nr : updatedNodes) {
+      // Scheduler will find out from the node, if at all.
+      // Relying on the RM to not allocate containers on an unhealthy node.
+      eventHandler.handle(new AMNodeEventStateChanged(nr, schedulerId));
+    }
+  }
+
+  public synchronized void appShutdownRequested(int schedulerId) {
+    // This can happen if the RM has been restarted. If it is in that state,
+    // this application must clean itself up.
+    LOG.info("App shutdown requested by scheduler {}", schedulerId);
+    sendEvent(new DAGAppMasterEvent(DAGAppMasterEventType.AM_REBOOT));
+  }
+
+  public synchronized void setApplicationRegistrationData(
+      int schedulerId,
+      Resource maxContainerCapability,
+      Map<ApplicationAccessType, String> appAcls, 
+      ByteBuffer clientAMSecretKey) {
+    this.appContext.getClusterInfo().setMaxContainerCapability(
+        maxContainerCapability);
+    this.appAcls = appAcls;
+    this.clientService.setClientAMSecretKey(clientAMSecretKey);
+  }
+
+  // Not synchronized to avoid deadlocks from TaskScheduler callbacks.
+  // TaskScheduler uses a separate thread for it's callbacks. Since this method
+  // returns a value which is required, the TaskScheduler wait for the call to
+  // complete and can hence lead to a deadlock if called from within a TSEH lock.
+  public AppFinalStatus getFinalAppStatus() {
+    FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
+    StringBuffer sb = new StringBuffer();
+    if (dagAppMaster == null) {
+      finishState = FinalApplicationStatus.UNDEFINED;
+      sb.append("App not yet initialized");
+    } else {
+      DAGAppMasterState appMasterState = dagAppMaster.getState();
+      if (appMasterState == DAGAppMasterState.SUCCEEDED) {
+        finishState = FinalApplicationStatus.SUCCEEDED;
+      } else if (appMasterState == DAGAppMasterState.KILLED
+          || (appMasterState == DAGAppMasterState.RUNNING && isSignalled)) {
+        finishState = FinalApplicationStatus.KILLED;
+      } else if (appMasterState == DAGAppMasterState.FAILED
+          || appMasterState == DAGAppMasterState.ERROR) {
+        finishState = FinalApplicationStatus.FAILED;
+      } else {
+        finishState = FinalApplicationStatus.UNDEFINED;
+      }
+      List<String> diagnostics = dagAppMaster.getDiagnostics();
+      if(diagnostics != null) {
+        for (String s : diagnostics) {
+          sb.append(s).append("\n");
+        }
+      }
+    }
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("Setting job diagnostics to " + sb.toString());
+    }
+
+    // if history url is set use the same, if historyUrl is set to "" then rm ui disables the
+    // history url
+    return new AppFinalStatus(finishState, sb.toString(), historyUrl);
+  }
+
+
+
+  // Not synchronized to avoid deadlocks from TaskScheduler callbacks.
+  // TaskScheduler uses a separate thread for it's callbacks. Since this method
+  // returns a value which is required, the TaskScheduler wait for the call to
+  // complete and can hence lead to a deadlock if called from within a TSEH lock.
+  public float getProgress(int schedulerId) {
+    // at this point allocate has been called and so node count must be available
+    // may change after YARN-1722
+    // This is a heartbeat in from the scheduler into the APP, and is being used to piggy-back and
+    // node updates from the cluster.
+
+    // Doubles as a mechanism to update node counts periodically. Hence schedulerId required.
+
+    // TODO Handle this in TEZ-2124. Need a way to know which scheduler is calling in.
+    int nodeCount = taskSchedulers[0].getClusterNodeCount();
+    if (nodeCount != cachedNodeCount) {
+      cachedNodeCount = nodeCount;
+      sendEvent(new AMNodeEventNodeCountUpdated(cachedNodeCount, schedulerId));
+    }
+    return dagAppMaster.getProgress();
+  }
+
+  public void onError(int schedulerId, Throwable t) {
+    LOG.info("Error reported by scheduler {} - {}", schedulerId, t);
+    sendEvent(new DAGAppMasterEventSchedulingServiceError(t));
+  }
+
+  public void dagCompleted() {
+    for (int i = 0 ; i < taskSchedulers.length ; i++) {
+      taskSchedulers[i].dagComplete();
+    }
+  }
+
+  public void dagSubmitted() {
+    // Nothing to do right now. Indicates that a new DAG has been submitted and
+    // the context has updated information.
+  }
+
+  public void preemptContainer(int schedulerId, ContainerId containerId) {
+    // TODO Why is this making a call back into the scheduler, when the call is originating from there.
+    // An AMContainer instance should already exist if an attempt is being made to preempt it
+    AMContainer amContainer = appContext.getAllContainers().get(containerId);
+    taskSchedulers[amContainer.getTaskSchedulerIdentifier()].deallocateContainer(containerId);
+    // Inform the Containers about completion.
+    sendEvent(new AMContainerEventCompleted(containerId, ContainerExitStatus.INVALID,
+        "Container preempted internally", TaskAttemptTerminationCause.INTERNAL_PREEMPTION));
+  }
+
+  public void setShouldUnregisterFlag() {
+    LOG.info("TaskScheduler notified that it should unregister from RM");
+    this.shouldUnregisterFlag.set(true);
+    for (int i = 0 ; i < taskSchedulers.length ; i++) {
+      if (this.taskSchedulers[i] != null) {
+        this.taskSchedulers[i].setShouldUnregister();
+      }
+    }
+  }
+
+  public ContainerSignatureMatcher getContainerSignatureMatcher() {
+    return containerSignatureMatcher;
+  }
+
+  public boolean hasUnregistered() {
+    boolean result = true;
+    for (int i = 0 ; i < taskSchedulers.length ; i++) {
+      result = result & this.taskSchedulers[i].hasUnregistered();
+      if (result == false) {
+        return result;
+      }
+    }
+    return result;
+  }
+
+  @VisibleForTesting
+  public String getHistoryUrl() {
+    Configuration config = this.appContext.getAMConf();
+    String historyUrl = "";
+
+    String loggingClass =  config.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "");
+    String historyUrlTemplate = config.get(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
+            TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE_DEFAULT);
+    String historyUrlBase = config.get(TezConfiguration.TEZ_HISTORY_URL_BASE, "");
+
+
+    if (loggingClass.equals("org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService") &&
+        !historyUrlTemplate.isEmpty() &&
+        !historyUrlBase.isEmpty()) {
+      // replace the placeholders, while tolerating extra or missing "/" in input. replace all
+      // instances of consecutive "/" with single (except for the http(s):// case
+      historyUrl = historyUrlTemplate
+          .replaceAll(APPLICATION_ID_PLACEHOLDER, appContext.getApplicationID().toString())
+          .replaceAll(HISTORY_URL_BASE, historyUrlBase)
+          .replaceAll("([^:])/{2,}", "$1/");
+
+      // make sure we have a valid scheme
+      if (!historyUrl.startsWith("http")) {
+        historyUrl = "http://" + historyUrl;
+      }
+    }
+
+    return historyUrl;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 99cec2b..69c21d4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -50,15 +50,15 @@ import org.apache.hadoop.yarn.util.Clock;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerHeartbeatHandler;
 import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.dag.event.DiagnosableEvent;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminatedBySystem;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminating;
 import org.apache.tez.dag.app.dag.event.TaskAttemptEventNodeFailed;
 import org.apache.tez.dag.app.rm.AMSchedulerEventDeallocateContainer;
-import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
-import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
+import org.apache.tez.dag.app.rm.ContainerLauncherLaunchRequestEvent;
+import org.apache.tez.dag.app.rm.ContainerLauncherStopRequestEvent;
 import org.apache.tez.dag.history.DAGHistoryEvent;
 import org.apache.tez.dag.history.HistoryEventHandler;
 import org.apache.tez.dag.history.events.ContainerStoppedEvent;
@@ -81,7 +81,7 @@ public class AMContainerImpl implements AMContainer {
   private final Container container;
   private final AppContext appContext;
   private final ContainerHeartbeatHandler containerHeartbeatHandler;
-  private final TaskAttemptListener taskAttemptListener;
+  private final TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
   protected final EventHandler eventHandler;
   private final ContainerSignatureMatcher signatureMatcher;
   private final int schedulerId;
@@ -308,7 +308,7 @@ public class AMContainerImpl implements AMContainer {
   // Attempting to use a container based purely on reosurces required, etc needs
   // additional change - JvmID, YarnChild, etc depend on TaskType.
   public AMContainerImpl(Container container, ContainerHeartbeatHandler chh,
-      TaskAttemptListener tal, ContainerSignatureMatcher signatureMatcher,
+      TaskCommunicatorManagerInterface tal, ContainerSignatureMatcher signatureMatcher,
       AppContext appContext, int schedulerId, int launcherId, int taskCommId) {
     ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
     this.readLock = rwLock.readLock();
@@ -319,7 +319,7 @@ public class AMContainerImpl implements AMContainer {
     this.signatureMatcher = signatureMatcher;
     this.appContext = appContext;
     this.containerHeartbeatHandler = chh;
-    this.taskAttemptListener = tal;
+    this.taskCommunicatorManagerInterface = tal;
     this.failedAssignments = new LinkedList<TezTaskAttemptID>();
     this.schedulerId = schedulerId;
     this.launcherId = launcherId;
@@ -466,7 +466,7 @@ public class AMContainerImpl implements AMContainer {
           containerContext.getLocalResources(),
           containerContext.getEnvironment(),
           containerContext.getJavaOpts(),
-          container.taskAttemptListener.getTaskCommunicator(container.taskCommId).getAddress(), containerContext.getCredentials(),
+          container.taskCommunicatorManagerInterface.getTaskCommunicator(container.taskCommId).getAddress(), containerContext.getCredentials(),
           container.appContext, container.container.getResource(),
           container.appContext.getAMConf());
 
@@ -1095,28 +1095,28 @@ public class AMContainerImpl implements AMContainer {
   }
 
   protected void sendStartRequestToNM(ContainerLaunchContext clc) {
-    sendEvent(new NMCommunicatorLaunchRequestEvent(clc, container, launcherId, schedulerId, taskCommId));
+    sendEvent(new ContainerLauncherLaunchRequestEvent(clc, container, launcherId, schedulerId, taskCommId));
   }
 
   protected void sendStopRequestToNM() {
-    sendEvent(new NMCommunicatorStopRequestEvent(containerId,
+    sendEvent(new ContainerLauncherStopRequestEvent(containerId,
         container.getNodeId(), container.getContainerToken(), launcherId, schedulerId, taskCommId));
   }
 
   protected void unregisterAttemptFromListener(TezTaskAttemptID attemptId, TaskAttemptEndReason endReason, String diagnostics) {
-    taskAttemptListener.unregisterTaskAttempt(attemptId, taskCommId, endReason, diagnostics);
+    taskCommunicatorManagerInterface.unregisterTaskAttempt(attemptId, taskCommId, endReason, diagnostics);
   }
 
   protected void registerAttemptWithListener(AMContainerTask amContainerTask) {
-    taskAttemptListener.registerTaskAttempt(amContainerTask, this.containerId, taskCommId);
+    taskCommunicatorManagerInterface.registerTaskAttempt(amContainerTask, this.containerId, taskCommId);
   }
 
   protected void registerWithTAListener() {
-    taskAttemptListener.registerRunningContainer(containerId, taskCommId);
+    taskCommunicatorManagerInterface.registerRunningContainer(containerId, taskCommId);
   }
 
   protected void unregisterFromTAListener(ContainerEndReason endReason, String diagnostics) {
-    this.taskAttemptListener.unregisterRunningContainer(containerId, taskCommId, endReason, diagnostics);
+    this.taskCommunicatorManagerInterface.unregisterRunningContainer(containerId, taskCommId, endReason, diagnostics);
   }
 
   protected void registerWithContainerListener() {

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
index fcb9eaf..ab43db1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
@@ -31,19 +31,19 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerHeartbeatHandler;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 
 public class AMContainerMap extends AbstractService implements EventHandler<AMContainerEvent> {
 
   private static final Logger LOG = LoggerFactory.getLogger(AMContainerMap.class);
 
   private final ContainerHeartbeatHandler chh;
-  private final TaskAttemptListener tal;
+  private final TaskCommunicatorManagerInterface tal;
   private final AppContext context;
   private final ContainerSignatureMatcher containerSignatureMatcher;
   private final ConcurrentHashMap<ContainerId, AMContainer> containerMap;
 
-  public AMContainerMap(ContainerHeartbeatHandler chh, TaskAttemptListener tal,
+  public AMContainerMap(ContainerHeartbeatHandler chh, TaskCommunicatorManagerInterface tal,
       ContainerSignatureMatcher containerSignatureMatcher, AppContext context) {
     super("AMContainerMaps");
     this.chh = chh;

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index b09eb86..3cab2da 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -60,8 +60,8 @@ import org.apache.tez.common.ContainerTask;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.app.launcher.ContainerLauncherRouter;
-import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
+import org.apache.tez.dag.app.launcher.ContainerLauncherManager;
+import org.apache.tez.dag.app.rm.ContainerLauncherEvent;
 import org.apache.tez.dag.history.HistoryEventHandler;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
@@ -122,13 +122,13 @@ public class MockDAGAppMaster extends DAGAppMaster {
   // It can be used to preempt the container for a given task
   public class MockContainerLauncher extends ContainerLauncher implements Runnable {
 
-    BlockingQueue<NMCommunicatorEvent> eventQueue = new LinkedBlockingQueue<NMCommunicatorEvent>();
+    BlockingQueue<ContainerLauncherEvent> eventQueue = new LinkedBlockingQueue<ContainerLauncherEvent>();
     Thread eventHandlingThread;
     ListeningExecutorService executorService;
     
     Map<ContainerId, ContainerData> containers = Maps.newConcurrentMap();
     ArrayBlockingQueue<Worker> workers;
-    TaskAttemptListenerImpTezDag taListener;
+    TaskCommunicatorManager taskCommunicatorManager;
     TezTaskCommunicatorImpl taskCommunicator;
     
     AtomicBoolean startScheduling = new AtomicBoolean(true);
@@ -187,8 +187,8 @@ public class MockDAGAppMaster extends DAGAppMaster {
     
     @Override
     public void start() throws Exception {
-      taListener = (TaskAttemptListenerImpTezDag) getTaskAttemptListener();
-      taskCommunicator = (TezTaskCommunicatorImpl) taListener.getTaskCommunicator(0);
+      taskCommunicatorManager = (TaskCommunicatorManager) getTaskCommunicatorManager();
+      taskCommunicator = (TezTaskCommunicatorImpl) taskCommunicatorManager.getTaskCommunicator(0);
       eventHandlingThread = new Thread(this);
       eventHandlingThread.start();
       ExecutorService rawExecutor = Executors.newFixedThreadPool(handlerConcurrency,
@@ -256,7 +256,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
     }
     
     public void preemptContainer(ContainerData cData) {
-      getTaskSchedulerEventHandler().containerCompleted(0, null,
+      getTaskSchedulerManager().containerCompleted(0, null,
           ContainerStatus.newInstance(cData.cId, null, "Preempted", ContainerExitStatus.PREEMPTED));
       cData.clear();
     }
@@ -495,7 +495,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
       throw new TezUncheckedException(e);
     }
     containerLauncherContext =
-        new ContainerLauncherContextImpl(getContext(), getTaskAttemptListener(), userPayload);
+        new ContainerLauncherContextImpl(getContext(), getTaskCommunicatorManager(), userPayload);
     containerLauncher = new MockContainerLauncher(launcherGoFlag, containerLauncherContext);
     shutdownHandler = new MockDAGAppMasterShutdownHandler();
     this.initFailFlag = initFailFlag;
@@ -507,10 +507,11 @@ public class MockDAGAppMaster extends DAGAppMaster {
 
   // use mock container launcher for tests
   @Override
-  protected ContainerLauncherRouter createContainerLauncherRouter(List<NamedEntityDescriptor> containerLauncherDescirptors,
-                                                                  boolean isLocal)
+  protected ContainerLauncherManager createContainerLauncherManager(
+      List<NamedEntityDescriptor> containerLauncherDescirptors,
+      boolean isLocal)
       throws UnknownHostException {
-    return new ContainerLauncherRouter(containerLauncher, getContext());
+    return new ContainerLauncherManager(containerLauncher, getContext());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
index 7584b4c..b0bc571 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
@@ -200,7 +200,7 @@ public class TestMockDAGAppMaster {
     mockLauncher.waitTillContainersLaunched();
     ContainerData cData = mockLauncher.getContainers().values().iterator().next();
     DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
-    mockApp.getTaskSchedulerEventHandler().preemptContainer(0, cData.cId);
+    mockApp.getTaskSchedulerManager().preemptContainer(0, cData.cId);
     
     mockLauncher.startScheduling(true);
     dagClient.waitForCompletion();


[3/7] tez git commit: TEZ-2708. Rename classes and variables post TEZ-2003 changes. (sseth)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 6ee741a..a8ba445 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -117,7 +117,7 @@ import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerHeartbeatHandler;
 import org.apache.tez.dag.app.MockClock;
 import org.apache.tez.dag.app.TaskAttemptEventInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.RootInputInitializerManager;
@@ -153,7 +153,7 @@ import org.apache.tez.dag.app.dag.event.VertexEventType;
 import org.apache.tez.dag.app.dag.impl.AMUserCodeException.Source;
 import org.apache.tez.dag.app.dag.impl.DAGImpl.VertexGroupInfo;
 import org.apache.tez.dag.app.dag.impl.TestVertexImpl.VertexManagerWithException.VMExceptionLocation;
-import org.apache.tez.dag.app.rm.TaskSchedulerEventHandler;
+import org.apache.tez.dag.app.rm.TaskSchedulerManager;
 import org.apache.tez.dag.app.rm.container.AMContainerMap;
 import org.apache.tez.dag.app.rm.container.ContainerContextMatcher;
 import org.apache.tez.dag.history.DAGHistoryEvent;
@@ -161,7 +161,6 @@ import org.apache.tez.dag.history.HistoryEventHandler;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.library.vertexmanager.InputReadyVertexManager;
 import org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager;
-import org.apache.tez.dag.records.TaskAttemptIdentifierImpl;
 import org.apache.tez.dag.records.TaskAttemptTerminationCause;
 import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -228,7 +227,7 @@ public class TestVertexImpl {
   private Map<String, VertexImpl> vertices;
   private Map<TezVertexID, VertexImpl> vertexIdMap;
   private DrainDispatcher dispatcher;
-  private TaskAttemptListener taskAttemptListener;
+  private TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
   private Clock clock = new SystemClock();
   private TaskHeartbeatHandler thh;
   private AppContext appContext;
@@ -2077,16 +2076,16 @@ public class TestVertexImpl {
       if (useCustomInitializer) {
         if (customInitializer == null) {
           v = new VertexImplWithControlledInitializerManager(vertexId, vPlan, vPlan.getName(), conf,
-              dispatcher.getEventHandler(), taskAttemptListener,
+              dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
               clock, thh, appContext, locationHint, dispatcher, updateTracker);
         } else {
           v = new VertexImplWithRunningInputInitializer(vertexId, vPlan, vPlan.getName(), conf,
-              dispatcher.getEventHandler(), taskAttemptListener,
+              dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
               clock, thh, appContext, locationHint, dispatcher, customInitializer, updateTracker);
         }
       } else {
         v = new VertexImpl(vertexId, vPlan, vPlan.getName(), conf,
-            dispatcher.getEventHandler(), taskAttemptListener,
+            dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
             clock, thh, true, appContext, locationHint, vertexGroups, taskSpecificLaunchCmdOption,
             updateTracker);
       }
@@ -2162,7 +2161,7 @@ public class TestVertexImpl {
     appContext = mock(AppContext.class);
     thh = mock(TaskHeartbeatHandler.class);
     historyEventHandler = mock(HistoryEventHandler.class);
-    TaskSchedulerEventHandler taskScheduler = mock(TaskSchedulerEventHandler.class);
+    TaskSchedulerManager taskScheduler = mock(TaskSchedulerManager.class);
     UserGroupInformation ugi;
     try {
       ugi = UserGroupInformation.getCurrentUser();
@@ -3227,7 +3226,7 @@ public class TestVertexImpl {
     when(container.getNodeId()).thenReturn(nid);
     when(container.getNodeHttpAddress()).thenReturn("localhost:0");
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appContext);
     containers.addContainerIfNew(container, 0, 0, 0);
     doReturn(containers).when(appContext).getAllContainers();
@@ -3262,7 +3261,7 @@ public class TestVertexImpl {
     when(container.getNodeId()).thenReturn(nid);
     when(container.getNodeHttpAddress()).thenReturn("localhost:0");
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appContext);
     containers.addContainerIfNew(container, 0, 0, 0);
     doReturn(containers).when(appContext).getAllContainers();
@@ -3298,7 +3297,7 @@ public class TestVertexImpl {
     when(container.getNodeId()).thenReturn(nid);
     when(container.getNodeHttpAddress()).thenReturn("localhost:0");
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appContext);
     containers.addContainerIfNew(container, 0, 0, 0);
     doReturn(containers).when(appContext).getAllContainers();
@@ -5179,7 +5178,7 @@ public class TestVertexImpl {
       vId = TezVertexID.getInstance(invalidDagId, 1);
       VertexPlan vPlan = invalidDagPlan.getVertex(0);
       VertexImpl v = new VertexImpl(vId, vPlan, vPlan.getName(), conf,
-          dispatcher.getEventHandler(), taskAttemptListener,
+          dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
           clock, thh, true, appContext, vertexLocationHint, null, taskSpecificLaunchCmdOption,
           updateTracker);
       v.setInputVertices(new HashMap());
@@ -5208,7 +5207,7 @@ public class TestVertexImpl {
                                                  VertexPlan vertexPlan, String vertexName,
                                                  Configuration conf,
                                                  EventHandler eventHandler,
-                                                 TaskAttemptListener taskAttemptListener,
+                                                 TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
                                                  Clock clock, TaskHeartbeatHandler thh,
                                                  AppContext appContext,
                                                  VertexLocationHint vertexLocationHint,
@@ -5216,7 +5215,7 @@ public class TestVertexImpl {
                                                  InputInitializer presetInitializer,
                                                  StateChangeNotifier updateTracker) {
       super(vertexId, vertexPlan, vertexName, conf, eventHandler,
-          taskAttemptListener, clock, thh, true,
+          taskCommunicatorManagerInterface, clock, thh, true,
           appContext, vertexLocationHint, null, taskSpecificLaunchCmdOption,
           updateTracker);
       this.presetInitializer = presetInitializer;
@@ -5248,14 +5247,14 @@ public class TestVertexImpl {
                                                       VertexPlan vertexPlan, String vertexName,
                                                       Configuration conf,
                                                       EventHandler eventHandler,
-                                                      TaskAttemptListener taskAttemptListener,
+                                                      TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
                                                       Clock clock, TaskHeartbeatHandler thh,
                                                       AppContext appContext,
                                                       VertexLocationHint vertexLocationHint,
                                                       DrainDispatcher dispatcher,
                                                       StateChangeNotifier updateTracker) {
       super(vertexId, vertexPlan, vertexName, conf, eventHandler,
-          taskAttemptListener, clock, thh, true,
+          taskCommunicatorManagerInterface, clock, thh, true,
           appContext, vertexLocationHint, null, taskSpecificLaunchCmdOption,
           updateTracker);
       this.dispatcher = dispatcher;

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
index 0e34f68..8bd288a 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
@@ -54,7 +54,7 @@ import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerContext;
 import org.apache.tez.dag.app.DAGAppMaster;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.StateChangeNotifier;
@@ -517,7 +517,7 @@ public class TestVertexImpl2 {
 
       vertex =
           new VertexImpl(TezVertexID.fromString("vertex_1418197758681_0001_1_00"), vertexPlan,
-              "testvertex", conf, mock(EventHandler.class), mock(TaskAttemptListener.class),
+              "testvertex", conf, mock(EventHandler.class), mock(TaskCommunicatorManagerInterface.class),
               mock(Clock.class), mock(TaskHeartbeatHandler.class), false, mockAppContext,
               VertexLocationHint.create(new LinkedList<TaskLocationHint>()), null,
               new TaskSpecificLaunchCmdOption(conf), mock(StateChangeNotifier.class));

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
index 0f532fb..e389d64 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
@@ -55,7 +55,7 @@ import org.apache.tez.dag.api.records.DAGProtos.TezEntityDescriptorProto;
 import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ClusterInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.DAGState;
 import org.apache.tez.dag.app.dag.Task;
@@ -434,7 +434,7 @@ public class TestVertexRecovery {
     DAGPlan dagPlan = createDAGPlan();
     dag =
         new DAGImpl(dagId, new Configuration(), dagPlan,
-            dispatcher.getEventHandler(), mock(TaskAttemptListener.class),
+            dispatcher.getEventHandler(), mock(TaskCommunicatorManagerInterface.class),
             new Credentials(), new SystemClock(), user,
             mock(TaskHeartbeatHandler.class), mockAppContext);
     when(mockAppContext.getCurrentDAG()).thenReturn(dag);
@@ -544,7 +544,7 @@ public class TestVertexRecovery {
     DAGPlan dagPlan = createDAGPlanSingleVertex();
     dag =
         new DAGImpl(dagId, new Configuration(), dagPlan,
-            dispatcher.getEventHandler(), mock(TaskAttemptListener.class),
+            dispatcher.getEventHandler(), mock(TaskCommunicatorManagerInterface.class),
             new Credentials(), new SystemClock(), user,
             mock(TaskHeartbeatHandler.class), mockAppContext);
     when(mockAppContext.getCurrentDAG()).thenReturn(dag);
@@ -924,7 +924,7 @@ public class TestVertexRecovery {
     DAGPlan dagPlan = createDAGPlanMR();
     dag =
         new DAGImpl(dagId, new Configuration(), dagPlan,
-            dispatcher.getEventHandler(), mock(TaskAttemptListener.class),
+            dispatcher.getEventHandler(), mock(TaskCommunicatorManagerInterface.class),
             new Credentials(), new SystemClock(), user,
             mock(TaskHeartbeatHandler.class), mockAppContext);
     when(mockAppContext.getCurrentDAG()).thenReturn(dag);
@@ -965,7 +965,7 @@ public class TestVertexRecovery {
     DAGPlan dagPlan = createDAGPlan();
     dag =
         new DAGImpl(dagId, new Configuration(), dagPlan,
-            dispatcher.getEventHandler(), mock(TaskAttemptListener.class),
+            dispatcher.getEventHandler(), mock(TaskCommunicatorManagerInterface.class),
             new Credentials(), new SystemClock(), user,
             mock(TaskHeartbeatHandler.class), mockAppContext);
     when(mockAppContext.getCurrentDAG()).thenReturn(dag);

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java
new file mode 100644
index 0000000..4b931d4
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.launcher;
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
+import org.apache.tez.dag.app.rm.ContainerLauncherLaunchRequestEvent;
+import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
+import org.apache.tez.serviceplugins.api.ContainerLauncher;
+import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
+import org.apache.tez.serviceplugins.api.ContainerStopRequest;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+public class TestContainerLauncherManager {
+
+  @Before
+  @After
+  public void reset() {
+    ContainerLaucherRouterForMultipleLauncherTest.reset();
+  }
+
+  @Test(timeout = 5000)
+  public void testNoLaunchersSpecified() throws IOException {
+
+    AppContext appContext = mock(AppContext.class);
+    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
+
+    try {
+
+      new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, null,
+          false);
+      fail("Expecting a failure without any launchers being specified");
+    } catch (IllegalArgumentException e) {
+
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testCustomLauncherSpecified() throws IOException {
+    Configuration conf = new Configuration(false);
+
+    AppContext appContext = mock(AppContext.class);
+    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
+
+    String customLauncherName = "customLauncher";
+    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
+    ByteBuffer bb = ByteBuffer.allocate(4);
+    bb.putInt(0, 3);
+    UserPayload customPayload = UserPayload.create(bb);
+    launcherDescriptors.add(
+        new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
+            .setUserPayload(customPayload));
+
+    ContainerLaucherRouterForMultipleLauncherTest clr =
+        new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
+            launcherDescriptors,
+            true);
+    try {
+      clr.init(conf);
+      clr.start();
+
+      assertEquals(1, clr.getNumContainerLaunchers());
+      assertFalse(clr.getYarnContainerLauncherCreated());
+      assertFalse(clr.getUberContainerLauncherCreated());
+      assertEquals(customLauncherName, clr.getContainerLauncherName(0));
+      assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
+    } finally {
+      clr.stop();
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testMultipleContainerLaunchers() throws IOException {
+    Configuration conf = new Configuration(false);
+    conf.set("testkey", "testvalue");
+    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+
+    AppContext appContext = mock(AppContext.class);
+    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
+
+    String customLauncherName = "customLauncher";
+    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
+    ByteBuffer bb = ByteBuffer.allocate(4);
+    bb.putInt(0, 3);
+    UserPayload customPayload = UserPayload.create(bb);
+    launcherDescriptors.add(
+        new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
+            .setUserPayload(customPayload));
+    launcherDescriptors
+        .add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+            .setUserPayload(userPayload));
+
+    ContainerLaucherRouterForMultipleLauncherTest clr =
+        new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
+            launcherDescriptors,
+            true);
+    try {
+      clr.init(conf);
+      clr.start();
+
+      assertEquals(2, clr.getNumContainerLaunchers());
+      assertTrue(clr.getYarnContainerLauncherCreated());
+      assertFalse(clr.getUberContainerLauncherCreated());
+      assertEquals(customLauncherName, clr.getContainerLauncherName(0));
+      assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
+
+      assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
+      Configuration confParsed = TezUtils
+          .createConfFromUserPayload(clr.getContainerLauncherContext(1).getInitialUserPayload());
+      assertEquals("testvalue", confParsed.get("testkey"));
+    } finally {
+      clr.stop();
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testEventRouting() throws Exception {
+    Configuration conf = new Configuration(false);
+    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+
+    AppContext appContext = mock(AppContext.class);
+    TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
+
+    String customLauncherName = "customLauncher";
+    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
+    ByteBuffer bb = ByteBuffer.allocate(4);
+    bb.putInt(0, 3);
+    UserPayload customPayload = UserPayload.create(bb);
+    launcherDescriptors.add(
+        new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
+            .setUserPayload(customPayload));
+    launcherDescriptors
+        .add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+            .setUserPayload(userPayload));
+
+    ContainerLaucherRouterForMultipleLauncherTest clr =
+        new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
+            launcherDescriptors,
+            true);
+    try {
+      clr.init(conf);
+      clr.start();
+
+      assertEquals(2, clr.getNumContainerLaunchers());
+      assertTrue(clr.getYarnContainerLauncherCreated());
+      assertFalse(clr.getUberContainerLauncherCreated());
+      assertEquals(customLauncherName, clr.getContainerLauncherName(0));
+      assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
+
+      verify(clr.getTestContainerLauncher(0)).initialize();
+      verify(clr.getTestContainerLauncher(0)).start();
+      verify(clr.getTestContainerLauncher(1)).initialize();
+      verify(clr.getTestContainerLauncher(1)).start();
+
+      ContainerLaunchContext clc1 = mock(ContainerLaunchContext.class);
+      Container container1 = mock(Container.class);
+
+      ContainerLaunchContext clc2 = mock(ContainerLaunchContext.class);
+      Container container2 = mock(Container.class);
+
+      ContainerLauncherLaunchRequestEvent launchRequestEvent1 =
+          new ContainerLauncherLaunchRequestEvent(clc1, container1, 0, 0, 0);
+      ContainerLauncherLaunchRequestEvent launchRequestEvent2 =
+          new ContainerLauncherLaunchRequestEvent(clc2, container2, 1, 0, 0);
+
+      clr.handle(launchRequestEvent1);
+
+
+      ArgumentCaptor<ContainerLaunchRequest> captor =
+          ArgumentCaptor.forClass(ContainerLaunchRequest.class);
+      verify(clr.getTestContainerLauncher(0)).launchContainer(captor.capture());
+      assertEquals(1, captor.getAllValues().size());
+      ContainerLaunchRequest launchRequest1 = captor.getValue();
+      assertEquals(clc1, launchRequest1.getContainerLaunchContext());
+
+      clr.handle(launchRequestEvent2);
+      captor = ArgumentCaptor.forClass(ContainerLaunchRequest.class);
+      verify(clr.getTestContainerLauncher(1)).launchContainer(captor.capture());
+      assertEquals(1, captor.getAllValues().size());
+      ContainerLaunchRequest launchRequest2 = captor.getValue();
+      assertEquals(clc2, launchRequest2.getContainerLaunchContext());
+
+    } finally {
+      clr.stop();
+      verify(clr.getTestContainerLauncher(0)).shutdown();
+      verify(clr.getTestContainerLauncher(1)).shutdown();
+    }
+  }
+
+  private static class ContainerLaucherRouterForMultipleLauncherTest
+      extends ContainerLauncherManager {
+
+    // All variables setup as static since methods being overridden are invoked by the ContainerLauncherRouter ctor,
+    // and regular variables will not be initialized at this point.
+    private static final AtomicInteger numContainerLaunchers = new AtomicInteger(0);
+    private static final Set<Integer> containerLauncherIndices = new HashSet<>();
+    private static final ContainerLauncher yarnContainerLauncher = mock(ContainerLauncher.class);
+    private static final ContainerLauncher uberContainerlauncher = mock(ContainerLauncher.class);
+    private static final AtomicBoolean yarnContainerLauncherCreated = new AtomicBoolean(false);
+    private static final AtomicBoolean uberContainerLauncherCreated = new AtomicBoolean(false);
+
+    private static final List<ContainerLauncherContext> containerLauncherContexts =
+        new LinkedList<>();
+    private static final List<String> containerLauncherNames = new LinkedList<>();
+    private static final List<ContainerLauncher> testContainerLaunchers = new LinkedList<>();
+
+
+    public static void reset() {
+      numContainerLaunchers.set(0);
+      containerLauncherIndices.clear();
+      yarnContainerLauncherCreated.set(false);
+      uberContainerLauncherCreated.set(false);
+      containerLauncherContexts.clear();
+      containerLauncherNames.clear();
+      testContainerLaunchers.clear();
+    }
+
+    public ContainerLaucherRouterForMultipleLauncherTest(AppContext context,
+                                                         TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+                                                         String workingDirectory,
+                                                         List<NamedEntityDescriptor> containerLauncherDescriptors,
+                                                         boolean isPureLocalMode) throws
+        UnknownHostException {
+      super(context, taskCommunicatorManagerInterface, workingDirectory,
+          containerLauncherDescriptors, isPureLocalMode);
+    }
+
+    @Override
+    ContainerLauncher createContainerLauncher(NamedEntityDescriptor containerLauncherDescriptor,
+                                              AppContext context,
+                                              ContainerLauncherContext containerLauncherContext,
+                                              TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+                                              String workingDirectory,
+                                              int containerLauncherIndex,
+                                              boolean isPureLocalMode) {
+      numContainerLaunchers.incrementAndGet();
+      boolean added = containerLauncherIndices.add(containerLauncherIndex);
+      assertTrue("Cannot add multiple launchers with the same index", added);
+      containerLauncherNames.add(containerLauncherDescriptor.getEntityName());
+      containerLauncherContexts.add(containerLauncherContext);
+      return super
+          .createContainerLauncher(containerLauncherDescriptor, context, containerLauncherContext,
+              taskCommunicatorManagerInterface, workingDirectory, containerLauncherIndex, isPureLocalMode);
+    }
+
+    @Override
+    ContainerLauncher createYarnContainerLauncher(
+        ContainerLauncherContext containerLauncherContext) {
+      yarnContainerLauncherCreated.set(true);
+      testContainerLaunchers.add(yarnContainerLauncher);
+      return yarnContainerLauncher;
+    }
+
+    @Override
+    ContainerLauncher createUberContainerLauncher(ContainerLauncherContext containerLauncherContext,
+                                                  AppContext context,
+                                                  TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+                                                  String workingDirectory,
+                                                  boolean isPureLocalMode) {
+      uberContainerLauncherCreated.set(true);
+      testContainerLaunchers.add(uberContainerlauncher);
+      return uberContainerlauncher;
+    }
+
+    @Override
+    ContainerLauncher createCustomContainerLauncher(
+        ContainerLauncherContext containerLauncherContext,
+        NamedEntityDescriptor containerLauncherDescriptor) {
+      ContainerLauncher spyLauncher = spy(super.createCustomContainerLauncher(
+          containerLauncherContext, containerLauncherDescriptor));
+      testContainerLaunchers.add(spyLauncher);
+      return spyLauncher;
+    }
+
+    public int getNumContainerLaunchers() {
+      return numContainerLaunchers.get();
+    }
+
+    public boolean getYarnContainerLauncherCreated() {
+      return yarnContainerLauncherCreated.get();
+    }
+
+    public boolean getUberContainerLauncherCreated() {
+      return uberContainerLauncherCreated.get();
+    }
+
+    public String getContainerLauncherName(int containerLauncherIndex) {
+      return containerLauncherNames.get(containerLauncherIndex);
+    }
+
+    public ContainerLauncher getTestContainerLauncher(int containerLauncherIndex) {
+      return testContainerLaunchers.get(containerLauncherIndex);
+    }
+
+    public ContainerLauncherContext getContainerLauncherContext(int containerLauncherIndex) {
+      return containerLauncherContexts.get(containerLauncherIndex);
+    }
+  }
+
+  private static class FakeContainerLauncher extends ContainerLauncher {
+
+    public FakeContainerLauncher(
+        ContainerLauncherContext containerLauncherContext) {
+      super(containerLauncherContext);
+    }
+
+    @Override
+    public void launchContainer(ContainerLaunchRequest launchRequest) {
+
+    }
+
+    @Override
+    public void stopContainer(ContainerStopRequest stopRequest) {
+
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherRouter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherRouter.java b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherRouter.java
deleted file mode 100644
index d0caf8c..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherRouter.java
+++ /dev/null
@@ -1,359 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.launcher;
-
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
-
-import java.io.IOException;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
-import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
-import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
-import org.apache.tez.serviceplugins.api.ContainerLauncher;
-import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
-import org.apache.tez.serviceplugins.api.ContainerStopRequest;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-
-public class TestContainerLauncherRouter {
-
-  @Before
-  @After
-  public void reset() {
-    ContainerLaucherRouterForMultipleLauncherTest.reset();
-  }
-
-  @Test(timeout = 5000)
-  public void testNoLaunchersSpecified() throws IOException {
-
-    AppContext appContext = mock(AppContext.class);
-    TaskAttemptListener tal = mock(TaskAttemptListener.class);
-
-    try {
-
-      new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, null,
-          false);
-      fail("Expecting a failure without any launchers being specified");
-    } catch (IllegalArgumentException e) {
-
-    }
-  }
-
-  @Test(timeout = 5000)
-  public void testCustomLauncherSpecified() throws IOException {
-    Configuration conf = new Configuration(false);
-
-    AppContext appContext = mock(AppContext.class);
-    TaskAttemptListener tal = mock(TaskAttemptListener.class);
-
-    String customLauncherName = "customLauncher";
-    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
-    ByteBuffer bb = ByteBuffer.allocate(4);
-    bb.putInt(0, 3);
-    UserPayload customPayload = UserPayload.create(bb);
-    launcherDescriptors.add(
-        new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
-            .setUserPayload(customPayload));
-
-    ContainerLaucherRouterForMultipleLauncherTest clr =
-        new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
-            launcherDescriptors,
-            true);
-    try {
-      clr.init(conf);
-      clr.start();
-
-      assertEquals(1, clr.getNumContainerLaunchers());
-      assertFalse(clr.getYarnContainerLauncherCreated());
-      assertFalse(clr.getUberContainerLauncherCreated());
-      assertEquals(customLauncherName, clr.getContainerLauncherName(0));
-      assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
-    } finally {
-      clr.stop();
-    }
-  }
-
-  @Test(timeout = 5000)
-  public void testMultipleContainerLaunchers() throws IOException {
-    Configuration conf = new Configuration(false);
-    conf.set("testkey", "testvalue");
-    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
-
-    AppContext appContext = mock(AppContext.class);
-    TaskAttemptListener tal = mock(TaskAttemptListener.class);
-
-    String customLauncherName = "customLauncher";
-    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
-    ByteBuffer bb = ByteBuffer.allocate(4);
-    bb.putInt(0, 3);
-    UserPayload customPayload = UserPayload.create(bb);
-    launcherDescriptors.add(
-        new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
-            .setUserPayload(customPayload));
-    launcherDescriptors
-        .add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
-            .setUserPayload(userPayload));
-
-    ContainerLaucherRouterForMultipleLauncherTest clr =
-        new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
-            launcherDescriptors,
-            true);
-    try {
-      clr.init(conf);
-      clr.start();
-
-      assertEquals(2, clr.getNumContainerLaunchers());
-      assertTrue(clr.getYarnContainerLauncherCreated());
-      assertFalse(clr.getUberContainerLauncherCreated());
-      assertEquals(customLauncherName, clr.getContainerLauncherName(0));
-      assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
-
-      assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
-      Configuration confParsed = TezUtils
-          .createConfFromUserPayload(clr.getContainerLauncherContext(1).getInitialUserPayload());
-      assertEquals("testvalue", confParsed.get("testkey"));
-    } finally {
-      clr.stop();
-    }
-  }
-
-  @Test(timeout = 5000)
-  public void testEventRouting() throws Exception {
-    Configuration conf = new Configuration(false);
-    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
-
-    AppContext appContext = mock(AppContext.class);
-    TaskAttemptListener tal = mock(TaskAttemptListener.class);
-
-    String customLauncherName = "customLauncher";
-    List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
-    ByteBuffer bb = ByteBuffer.allocate(4);
-    bb.putInt(0, 3);
-    UserPayload customPayload = UserPayload.create(bb);
-    launcherDescriptors.add(
-        new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
-            .setUserPayload(customPayload));
-    launcherDescriptors
-        .add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
-            .setUserPayload(userPayload));
-
-    ContainerLaucherRouterForMultipleLauncherTest clr =
-        new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
-            launcherDescriptors,
-            true);
-    try {
-      clr.init(conf);
-      clr.start();
-
-      assertEquals(2, clr.getNumContainerLaunchers());
-      assertTrue(clr.getYarnContainerLauncherCreated());
-      assertFalse(clr.getUberContainerLauncherCreated());
-      assertEquals(customLauncherName, clr.getContainerLauncherName(0));
-      assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
-
-      verify(clr.getTestContainerLauncher(0)).initialize();
-      verify(clr.getTestContainerLauncher(0)).start();
-      verify(clr.getTestContainerLauncher(1)).initialize();
-      verify(clr.getTestContainerLauncher(1)).start();
-
-      ContainerLaunchContext clc1 = mock(ContainerLaunchContext.class);
-      Container container1 = mock(Container.class);
-
-      ContainerLaunchContext clc2 = mock(ContainerLaunchContext.class);
-      Container container2 = mock(Container.class);
-
-      NMCommunicatorLaunchRequestEvent launchRequestEvent1 =
-          new NMCommunicatorLaunchRequestEvent(clc1, container1, 0, 0, 0);
-      NMCommunicatorLaunchRequestEvent launchRequestEvent2 =
-          new NMCommunicatorLaunchRequestEvent(clc2, container2, 1, 0, 0);
-
-      clr.handle(launchRequestEvent1);
-
-
-      ArgumentCaptor<ContainerLaunchRequest> captor =
-          ArgumentCaptor.forClass(ContainerLaunchRequest.class);
-      verify(clr.getTestContainerLauncher(0)).launchContainer(captor.capture());
-      assertEquals(1, captor.getAllValues().size());
-      ContainerLaunchRequest launchRequest1 = captor.getValue();
-      assertEquals(clc1, launchRequest1.getContainerLaunchContext());
-
-      clr.handle(launchRequestEvent2);
-      captor = ArgumentCaptor.forClass(ContainerLaunchRequest.class);
-      verify(clr.getTestContainerLauncher(1)).launchContainer(captor.capture());
-      assertEquals(1, captor.getAllValues().size());
-      ContainerLaunchRequest launchRequest2 = captor.getValue();
-      assertEquals(clc2, launchRequest2.getContainerLaunchContext());
-
-    } finally {
-      clr.stop();
-      verify(clr.getTestContainerLauncher(0)).shutdown();
-      verify(clr.getTestContainerLauncher(1)).shutdown();
-    }
-  }
-
-  private static class ContainerLaucherRouterForMultipleLauncherTest
-      extends ContainerLauncherRouter {
-
-    // All variables setup as static since methods being overridden are invoked by the ContainerLauncherRouter ctor,
-    // and regular variables will not be initialized at this point.
-    private static final AtomicInteger numContainerLaunchers = new AtomicInteger(0);
-    private static final Set<Integer> containerLauncherIndices = new HashSet<>();
-    private static final ContainerLauncher yarnContainerLauncher = mock(ContainerLauncher.class);
-    private static final ContainerLauncher uberContainerlauncher = mock(ContainerLauncher.class);
-    private static final AtomicBoolean yarnContainerLauncherCreated = new AtomicBoolean(false);
-    private static final AtomicBoolean uberContainerLauncherCreated = new AtomicBoolean(false);
-
-    private static final List<ContainerLauncherContext> containerLauncherContexts =
-        new LinkedList<>();
-    private static final List<String> containerLauncherNames = new LinkedList<>();
-    private static final List<ContainerLauncher> testContainerLaunchers = new LinkedList<>();
-
-
-    public static void reset() {
-      numContainerLaunchers.set(0);
-      containerLauncherIndices.clear();
-      yarnContainerLauncherCreated.set(false);
-      uberContainerLauncherCreated.set(false);
-      containerLauncherContexts.clear();
-      containerLauncherNames.clear();
-      testContainerLaunchers.clear();
-    }
-
-    public ContainerLaucherRouterForMultipleLauncherTest(AppContext context,
-                                                         TaskAttemptListener taskAttemptListener,
-                                                         String workingDirectory,
-                                                         List<NamedEntityDescriptor> containerLauncherDescriptors,
-                                                         boolean isPureLocalMode) throws
-        UnknownHostException {
-      super(context, taskAttemptListener, workingDirectory,
-          containerLauncherDescriptors, isPureLocalMode);
-    }
-
-    @Override
-    ContainerLauncher createContainerLauncher(NamedEntityDescriptor containerLauncherDescriptor,
-                                              AppContext context,
-                                              ContainerLauncherContext containerLauncherContext,
-                                              TaskAttemptListener taskAttemptListener,
-                                              String workingDirectory,
-                                              int containerLauncherIndex,
-                                              boolean isPureLocalMode) {
-      numContainerLaunchers.incrementAndGet();
-      boolean added = containerLauncherIndices.add(containerLauncherIndex);
-      assertTrue("Cannot add multiple launchers with the same index", added);
-      containerLauncherNames.add(containerLauncherDescriptor.getEntityName());
-      containerLauncherContexts.add(containerLauncherContext);
-      return super
-          .createContainerLauncher(containerLauncherDescriptor, context, containerLauncherContext,
-              taskAttemptListener, workingDirectory, containerLauncherIndex, isPureLocalMode);
-    }
-
-    @Override
-    ContainerLauncher createYarnContainerLauncher(
-        ContainerLauncherContext containerLauncherContext) {
-      yarnContainerLauncherCreated.set(true);
-      testContainerLaunchers.add(yarnContainerLauncher);
-      return yarnContainerLauncher;
-    }
-
-    @Override
-    ContainerLauncher createUberContainerLauncher(ContainerLauncherContext containerLauncherContext,
-                                                  AppContext context,
-                                                  TaskAttemptListener taskAttemptListener,
-                                                  String workingDirectory,
-                                                  boolean isPureLocalMode) {
-      uberContainerLauncherCreated.set(true);
-      testContainerLaunchers.add(uberContainerlauncher);
-      return uberContainerlauncher;
-    }
-
-    @Override
-    ContainerLauncher createCustomContainerLauncher(
-        ContainerLauncherContext containerLauncherContext,
-        NamedEntityDescriptor containerLauncherDescriptor) {
-      ContainerLauncher spyLauncher = spy(super.createCustomContainerLauncher(
-          containerLauncherContext, containerLauncherDescriptor));
-      testContainerLaunchers.add(spyLauncher);
-      return spyLauncher;
-    }
-
-    public int getNumContainerLaunchers() {
-      return numContainerLaunchers.get();
-    }
-
-    public boolean getYarnContainerLauncherCreated() {
-      return yarnContainerLauncherCreated.get();
-    }
-
-    public boolean getUberContainerLauncherCreated() {
-      return uberContainerLauncherCreated.get();
-    }
-
-    public String getContainerLauncherName(int containerLauncherIndex) {
-      return containerLauncherNames.get(containerLauncherIndex);
-    }
-
-    public ContainerLauncher getTestContainerLauncher(int containerLauncherIndex) {
-      return testContainerLaunchers.get(containerLauncherIndex);
-    }
-
-    public ContainerLauncherContext getContainerLauncherContext(int containerLauncherIndex) {
-      return containerLauncherContexts.get(containerLauncherIndex);
-    }
-  }
-
-  private static class FakeContainerLauncher extends ContainerLauncher {
-
-    public FakeContainerLauncher(
-        ContainerLauncherContext containerLauncherContext) {
-      super(containerLauncherContext);
-    }
-
-    @Override
-    public void launchContainer(ContainerLaunchRequest launchRequest) {
-
-    }
-
-    @Override
-    public void stopContainer(ContainerStopRequest stopRequest) {
-
-    }
-  }
-
-}


[4/7] tez git commit: TEZ-2708. Rename classes and variables post TEZ-2003 changes. (sseth)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
deleted file mode 100644
index 5159aff..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.common.ContainerContext;
-import org.apache.tez.common.ContainerTask;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.common.security.JobTokenIdentifier;
-import org.apache.tez.common.security.JobTokenSecretManager;
-import org.apache.tez.common.security.TokenCache;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TaskCommunicator;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.serviceplugins.api.ContainerEndReason;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.dag.api.TaskHeartbeatRequest;
-import org.apache.tez.dag.api.TaskHeartbeatResponse;
-import org.apache.tez.dag.api.TezException;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.tez.common.TezTaskUmbilicalProtocol;
-import org.apache.tez.dag.api.TaskCommunicatorContext;
-import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.app.dag.Vertex;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
-import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
-import org.apache.tez.dag.app.dag.event.VertexEventType;
-import org.apache.tez.dag.app.rm.container.AMContainer;
-import org.apache.tez.dag.app.rm.container.AMContainerMap;
-import org.apache.tez.dag.app.rm.container.AMContainerTask;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-import org.apache.tez.dag.records.TezVertexID;
-import org.apache.tez.runtime.api.events.DataMovementEvent;
-import org.apache.tez.runtime.api.events.TaskAttemptCompletedEvent;
-import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent;
-import org.apache.tez.runtime.api.impl.EventType;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.runtime.api.impl.TezEvent;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-
-@SuppressWarnings("unchecked")
-// TODO TEZ-2003 (post) TEZ-2696 Rename to TestTezTaskCommunicator
-public class TestTaskAttemptListenerImplTezDag {
-  private ApplicationId appId;
-  private ApplicationAttemptId appAttemptId;
-  private AppContext appContext;
-  Credentials credentials;
-  AMContainerMap amContainerMap;
-  EventHandler eventHandler;
-  DAG dag;
-  TaskAttemptListenerImpTezDag taskAttemptListener;
-  ContainerTask containerTask;
-  AMContainerTask amContainerTask;
-  TaskSpec taskSpec;
-
-  TezVertexID vertexID;
-  TezTaskID taskID;
-  TezTaskAttemptID taskAttemptID;
-
-  @Before
-  public void setUp() {
-    appId = ApplicationId.newInstance(1000, 1);
-    appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
-    dag = mock(DAG.class);
-    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
-    vertexID = TezVertexID.getInstance(dagID, 1);
-    taskID = TezTaskID.getInstance(vertexID, 1);
-    taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
-    credentials = new Credentials();
-
-    amContainerMap = mock(AMContainerMap.class);
-    Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
-
-    eventHandler = mock(EventHandler.class);
-
-    MockClock clock = new MockClock();
-    
-    appContext = mock(AppContext.class);
-    doReturn(eventHandler).when(appContext).getEventHandler();
-    doReturn(dag).when(appContext).getCurrentDAG();
-    doReturn(appAcls).when(appContext).getApplicationACLs();
-    doReturn(amContainerMap).when(appContext).getAllContainers();
-    doReturn(clock).when(appContext).getClock();
-    
-    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
-    doReturn(credentials).when(appContext).getAppCredentials();
-    NodeId nodeId = NodeId.newInstance("localhost", 0);
-    AMContainer amContainer = mock(AMContainer.class);
-    Container container = mock(Container.class);
-    doReturn(nodeId).when(container).getNodeId();
-    doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
-    doReturn(container).when(amContainer).getContainer();
-
-    Configuration conf = new TezConfiguration();
-    UserPayload defaultPayload;
-    try {
-      defaultPayload = TezUtils.createUserPayloadFromConf(conf);
-    } catch (IOException e) {
-      throw new TezUncheckedException(e);
-    }
-    taskAttemptListener = new TaskAttemptListenerImplForTest(appContext,
-        mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class),
-        Lists.newArrayList(
-            new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
-                .setUserPayload(defaultPayload)));
-
-    taskSpec = mock(TaskSpec.class);
-    doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();
-    amContainerTask = new AMContainerTask(taskSpec, null, null, false, 0);
-    containerTask = null;
-  }
-
-  @Test(timeout = 5000)
-  public void testGetTask() throws IOException {
-
-    TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
-    TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
-
-    ContainerId containerId1 = createContainerId(appId, 1);
-    ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
-    containerTask = tezUmbilical.getTask(containerContext1);
-    assertTrue(containerTask.shouldDie());
-
-    ContainerId containerId2 = createContainerId(appId, 2);
-    ContainerContext containerContext2 = new ContainerContext(containerId2.toString());
-    taskAttemptListener.registerRunningContainer(containerId2, 0);
-    containerTask = tezUmbilical.getTask(containerContext2);
-    assertNull(containerTask);
-
-    // Valid task registered
-    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId2, 0);
-    containerTask = tezUmbilical.getTask(containerContext2);
-    assertFalse(containerTask.shouldDie());
-    assertEquals(taskSpec, containerTask.getTaskSpec());
-
-    // Task unregistered. Should respond to heartbeats
-    taskAttemptListener.unregisterTaskAttempt(taskAttemptID, 0, TaskAttemptEndReason.OTHER, null);
-    containerTask = tezUmbilical.getTask(containerContext2);
-    assertNull(containerTask);
-
-    // Container unregistered. Should send a shouldDie = true
-    taskAttemptListener.unregisterRunningContainer(containerId2, 0, ContainerEndReason.OTHER, null);
-    containerTask = tezUmbilical.getTask(containerContext2);
-    assertTrue(containerTask.shouldDie());
-
-    ContainerId containerId3 = createContainerId(appId, 3);
-    ContainerContext containerContext3 = new ContainerContext(containerId3.toString());
-    taskAttemptListener.registerRunningContainer(containerId3, 0);
-
-    // Register task to container3, followed by unregistering container 3 all together
-    TaskSpec taskSpec2 = mock(TaskSpec.class);
-    TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
-    doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
-    AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec, null, null, false, 0);
-    taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId3, 0);
-    taskAttemptListener.unregisterRunningContainer(containerId3, 0, ContainerEndReason.OTHER, null);
-    containerTask = tezUmbilical.getTask(containerContext3);
-    assertTrue(containerTask.shouldDie());
-  }
-
-  @Test(timeout = 5000)
-  public void testGetTaskMultiplePulls() throws IOException {
-    TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
-    TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
-
-    ContainerId containerId1 = createContainerId(appId, 1);
-
-    ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
-    taskAttemptListener.registerRunningContainer(containerId1, 0);
-    containerTask = tezUmbilical.getTask(containerContext1);
-    assertNull(containerTask);
-
-    // Register task
-    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId1, 0);
-    containerTask = tezUmbilical.getTask(containerContext1);
-    assertFalse(containerTask.shouldDie());
-    assertEquals(taskSpec, containerTask.getTaskSpec());
-
-    // Try pulling again - simulates re-use pull
-    containerTask = tezUmbilical.getTask(containerContext1);
-    assertNull(containerTask);
-  }
-
-  @Test (timeout = 5000)
-  public void testTaskEventRouting() throws Exception {
-    List<TezEvent> events =  Arrays.asList(
-      new TezEvent(new TaskStatusUpdateEvent(null, 0.0f, null), null),
-      new TezEvent(DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0])), null),
-      new TezEvent(new TaskAttemptCompletedEvent(), null)
-    );
-
-    generateHeartbeat(events, 0, 1, 0, new ArrayList<TezEvent>());
-
-    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
-    verify(eventHandler, times(2)).handle(arg.capture());
-    final List<Event> argAllValues = arg.getAllValues();
-
-    final Event statusUpdateEvent = argAllValues.get(0);
-    assertEquals("First event should be status update", TaskAttemptEventType.TA_STATUS_UPDATE,
-        statusUpdateEvent.getType());
-
-    final Event vertexEvent = argAllValues.get(1);
-    final VertexEventRouteEvent vertexRouteEvent = (VertexEventRouteEvent)vertexEvent;
-    assertEquals("First event should be routed to vertex", VertexEventType.V_ROUTE_EVENT,
-        vertexEvent.getType());
-    assertEquals(EventType.DATA_MOVEMENT_EVENT,
-        vertexRouteEvent.getEvents().get(0).getEventType());
-    assertEquals(EventType.TASK_ATTEMPT_COMPLETED_EVENT,
-        vertexRouteEvent.getEvents().get(1).getEventType());
-
-  }
-
-  @Test (timeout = 5000)
-  public void testTaskEventRoutingTaskAttemptOnly() throws Exception {
-    List<TezEvent> events = Arrays.asList(
-      new TezEvent(new TaskAttemptCompletedEvent(), null)
-    );
-    generateHeartbeat(events, 0, 1, 0, new ArrayList<TezEvent>());
-
-    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
-    verify(eventHandler, times(1)).handle(arg.capture());
-    final List<Event> argAllValues = arg.getAllValues();
-
-    final Event event = argAllValues.get(0);
-    assertEquals("only event should be route event", VertexEventType.V_ROUTE_EVENT,
-        event.getType());
-  }
-  
-  @Test (timeout = 5000)
-  public void testTaskHeartbeatResponse() throws Exception {
-    List<TezEvent> events = new ArrayList<TezEvent>();
-    List<TezEvent> eventsToSend = new ArrayList<TezEvent>();
-    TaskHeartbeatResponse response = generateHeartbeat(events, 0, 1, 2, eventsToSend);
-    
-    assertEquals(2, response.getNextFromEventId());
-    assertEquals(eventsToSend, response.getEvents());
-  }
-
-  //try 10 times to allocate random port, fail it if no one is succeed.
-  @Test (timeout = 5000)
-  public void testPortRange() {
-    boolean succeedToAllocate = false;
-    Random rand = new Random();
-    for (int i = 0; i < 10; ++i) {
-      int nextPort = 1024 + rand.nextInt(65535 - 1024);
-      if (testPortRange(nextPort)) {
-        succeedToAllocate = true;
-        break;
-      }
-    }
-    if (!succeedToAllocate) {
-      fail("Can not allocate free port even in 10 iterations for TaskAttemptListener");
-    }
-  }
-
-  // TODO TEZ-2003 Move this into TestTezTaskCommunicator. Potentially other tests as well.
-  @Test (timeout= 5000)
-  public void testPortRange_NotSpecified() throws IOException {
-    Configuration conf = new Configuration();
-    JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(
-        "fakeIdentifier"));
-    Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier,
-        new JobTokenSecretManager());
-    sessionToken.setService(identifier.getJobId());
-    TokenCache.setSessionToken(sessionToken, credentials);
-    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
-    taskAttemptListener = new TaskAttemptListenerImpTezDag(appContext,
-        mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(
-        new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
-            .setUserPayload(userPayload)));
-    // no exception happen, should started properly
-    taskAttemptListener.init(conf);
-    taskAttemptListener.start();
-  }
-
-  private boolean testPortRange(int port) {
-    boolean succeedToAllocate = true;
-    try {
-      Configuration conf = new Configuration();
-      
-      JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(
-          "fakeIdentifier"));
-      Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier,
-          new JobTokenSecretManager());
-      sessionToken.setService(identifier.getJobId());
-      TokenCache.setSessionToken(sessionToken, credentials);
-
-      conf.set(TezConfiguration.TEZ_AM_TASK_AM_PORT_RANGE, port + "-" + port);
-      UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
-
-      taskAttemptListener = new TaskAttemptListenerImpTezDag(appContext,
-          mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists
-          .newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
-              .setUserPayload(userPayload)));
-      taskAttemptListener.init(conf);
-      taskAttemptListener.start();
-      int resultedPort = taskAttemptListener.getTaskCommunicator(0).getAddress().getPort();
-      assertEquals(port, resultedPort);
-    } catch (Exception e) {
-      succeedToAllocate = false;
-    } finally {
-      if (taskAttemptListener != null) {
-        try {
-          taskAttemptListener.close();
-        } catch (IOException e) {
-          e.printStackTrace();
-          fail("fail to stop TaskAttemptListener");
-        }
-      }
-    }
-    return succeedToAllocate;
-  }
-
-  private TaskHeartbeatResponse generateHeartbeat(List<TezEvent> events,
-      int fromEventId, int maxEvents, int nextFromEventId,
-      List<TezEvent> sendEvents) throws IOException, TezException {
-    ContainerId containerId = createContainerId(appId, 1);
-    Vertex vertex = mock(Vertex.class);
-
-    doReturn(vertex).when(dag).getVertex(vertexID);
-    doReturn("test_vertex").when(vertex).getName();
-    TaskAttemptEventInfo eventInfo = new TaskAttemptEventInfo(nextFromEventId, sendEvents, 0);
-    doReturn(eventInfo).when(vertex).getTaskAttemptTezEvents(taskAttemptID, fromEventId, 0, maxEvents);
-
-    taskAttemptListener.registerRunningContainer(containerId, 0);
-    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId, 0);
-
-    TaskHeartbeatRequest request = mock(TaskHeartbeatRequest.class);
-    doReturn(containerId.toString()).when(request).getContainerIdentifier();
-    doReturn(containerId.toString()).when(request).getContainerIdentifier();
-    doReturn(taskAttemptID).when(request).getTaskAttemptId();
-    doReturn(events).when(request).getEvents();
-    doReturn(maxEvents).when(request).getMaxEvents();
-    doReturn(fromEventId).when(request).getStartIndex();
-
-    return taskAttemptListener.heartbeat(request);
-  }
-
-
-  private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
-    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
-    return ContainerId.newInstance(appAttemptId, containerIdx);
-  }
-
-  private static class TaskAttemptListenerImplForTest extends TaskAttemptListenerImpTezDag {
-
-    public TaskAttemptListenerImplForTest(AppContext context,
-                                          TaskHeartbeatHandler thh,
-                                          ContainerHeartbeatHandler chh,
-                                          List<NamedEntityDescriptor> taskCommDescriptors) {
-      super(context, thh, chh, taskCommDescriptors);
-    }
-
-    @Override
-    TaskCommunicator createDefaultTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
-      return new TezTaskCommunicatorImplForTest(taskCommunicatorContext);
-    }
-
-  }
-
-  private static class TezTaskCommunicatorImplForTest extends TezTaskCommunicatorImpl {
-
-    public TezTaskCommunicatorImplForTest(
-        TaskCommunicatorContext taskCommunicatorContext) {
-      super(taskCommunicatorContext);
-    }
-
-    @Override
-    protected void startRpcServer() {
-    }
-
-    @Override
-    protected void stopRpcServer() {
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java
deleted file mode 100644
index 74468f2..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptKilled;
-import org.apache.tez.dag.app.rm.container.AMContainer;
-import org.apache.tez.dag.app.rm.container.AMContainerMap;
-import org.apache.tez.dag.app.rm.container.AMContainerTask;
-import org.apache.tez.dag.records.TaskAttemptTerminationCause;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-
-// TODO TEZ-2003. Rename to TestTaskAttemptListener | whatever TaskAttemptListener is renamed to.
-public class TestTaskAttemptListenerImplTezDag2 {
-
-  @Test(timeout = 5000)
-  public void testTaskAttemptFailedKilled() throws IOException {
-    ApplicationId appId = ApplicationId.newInstance(1000, 1);
-    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
-    Credentials credentials = new Credentials();
-    AppContext appContext = mock(AppContext.class);
-    EventHandler eventHandler = mock(EventHandler.class);
-    DAG dag = mock(DAG.class);
-    AMContainerMap amContainerMap = mock(AMContainerMap.class);
-    Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
-    doReturn(eventHandler).when(appContext).getEventHandler();
-    doReturn(dag).when(appContext).getCurrentDAG();
-    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
-    doReturn(credentials).when(appContext).getAppCredentials();
-    doReturn(appAcls).when(appContext).getApplicationACLs();
-    doReturn(amContainerMap).when(appContext).getAllContainers();
-    NodeId nodeId = NodeId.newInstance("localhost", 0);
-    AMContainer amContainer = mock(AMContainer.class);
-    Container container = mock(Container.class);
-    doReturn(nodeId).when(container).getNodeId();
-    doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
-    doReturn(container).when(amContainer).getContainer();
-
-    Configuration conf = new TezConfiguration();
-    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
-    TaskAttemptListenerImpTezDag taskAttemptListener =
-        new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
-            mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(
-            TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload)));
-
-    TaskSpec taskSpec1 = mock(TaskSpec.class);
-    TezTaskAttemptID taskAttemptId1 = mock(TezTaskAttemptID.class);
-    doReturn(taskAttemptId1).when(taskSpec1).getTaskAttemptID();
-    AMContainerTask amContainerTask1 = new AMContainerTask(taskSpec1, null, null, false, 10);
-
-    TaskSpec taskSpec2 = mock(TaskSpec.class);
-    TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
-    doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
-    AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec2, null, null, false, 10);
-
-    ContainerId containerId1 = createContainerId(appId, 1);
-    taskAttemptListener.registerRunningContainer(containerId1, 0);
-    taskAttemptListener.registerTaskAttempt(amContainerTask1, containerId1, 0);
-    ContainerId containerId2 = createContainerId(appId, 2);
-    taskAttemptListener.registerRunningContainer(containerId2, 0);
-    taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId2, 0);
-
-
-    taskAttemptListener
-        .taskFailed(taskAttemptId1, TaskAttemptEndReason.COMMUNICATION_ERROR, "Diagnostics1");
-    taskAttemptListener
-        .taskKilled(taskAttemptId2, TaskAttemptEndReason.EXECUTOR_BUSY, "Diagnostics2");
-
-    ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
-    verify(eventHandler, times(2)).handle(argumentCaptor.capture());
-    assertTrue(argumentCaptor.getAllValues().get(0) instanceof TaskAttemptEventAttemptFailed);
-    assertTrue(argumentCaptor.getAllValues().get(1) instanceof TaskAttemptEventAttemptKilled);
-    TaskAttemptEventAttemptFailed failedEvent =
-        (TaskAttemptEventAttemptFailed) argumentCaptor.getAllValues().get(0);
-    TaskAttemptEventAttemptKilled killedEvent =
-        (TaskAttemptEventAttemptKilled) argumentCaptor.getAllValues().get(1);
-
-    assertEquals("Diagnostics1", failedEvent.getDiagnosticInfo());
-    assertEquals(TaskAttemptTerminationCause.COMMUNICATION_ERROR,
-        failedEvent.getTerminationCause());
-
-    assertEquals("Diagnostics2", killedEvent.getDiagnosticInfo());
-    assertEquals(TaskAttemptTerminationCause.SERVICE_BUSY, killedEvent.getTerminationCause());
-    // TODO TEZ-2003. Verify unregistration from the registered list
-  }
-
-  private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
-    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
-    ContainerId containerId = ContainerId.newInstance(appAttemptId, containerIdx);
-    return containerId;
-  }
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorContextImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorContextImpl.java
index 1545eb4..5222a2d 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorContextImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorContextImpl.java
@@ -34,7 +34,7 @@ public class TestTaskCommunicatorContextImpl {
   @Test(timeout = 5000)
   public void testIsKnownContainer() {
     AppContext appContext = mock(AppContext.class);
-    TaskAttemptListenerImpTezDag tal = mock(TaskAttemptListenerImpTezDag.class);
+    TaskCommunicatorManager tal = mock(TaskCommunicatorManager.class);
 
     AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class), tal, mock(
         ContainerSignatureMatcher.class), appContext);

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java
index 4f68fab..be7adde 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java
@@ -220,7 +220,7 @@ public class TestTaskCommunicatorManager {
   }
 
 
-  static class TaskCommManagerForMultipleCommTest extends TaskAttemptListenerImpTezDag {
+  static class TaskCommManagerForMultipleCommTest extends TaskCommunicatorManager {
 
     // All variables setup as static since methods being overridden are invoked by the ContainerLauncherRouter ctor,
     // and regular variables will not be initialized at this point.

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java
new file mode 100644
index 0000000..e8ce429
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java
@@ -0,0 +1,425 @@
+/*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.app;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.ContainerContext;
+import org.apache.tez.common.ContainerTask;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TaskCommunicator;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.serviceplugins.api.ContainerEndReason;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.dag.api.TaskHeartbeatRequest;
+import org.apache.tez.dag.api.TaskHeartbeatResponse;
+import org.apache.tez.dag.api.TezException;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
+import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
+import org.apache.tez.dag.app.dag.event.VertexEventType;
+import org.apache.tez.dag.app.rm.container.AMContainer;
+import org.apache.tez.dag.app.rm.container.AMContainerMap;
+import org.apache.tez.dag.app.rm.container.AMContainerTask;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.api.events.DataMovementEvent;
+import org.apache.tez.runtime.api.events.TaskAttemptCompletedEvent;
+import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent;
+import org.apache.tez.runtime.api.impl.EventType;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.runtime.api.impl.TezEvent;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+@SuppressWarnings("unchecked")
+public class TestTaskCommunicatorManager1 {
+  private ApplicationId appId;
+  private ApplicationAttemptId appAttemptId;
+  private AppContext appContext;
+  Credentials credentials;
+  AMContainerMap amContainerMap;
+  EventHandler eventHandler;
+  DAG dag;
+  TaskCommunicatorManager taskAttemptListener;
+  ContainerTask containerTask;
+  AMContainerTask amContainerTask;
+  TaskSpec taskSpec;
+
+  TezVertexID vertexID;
+  TezTaskID taskID;
+  TezTaskAttemptID taskAttemptID;
+
+  @Before
+  public void setUp() {
+    appId = ApplicationId.newInstance(1000, 1);
+    appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+    dag = mock(DAG.class);
+    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
+    vertexID = TezVertexID.getInstance(dagID, 1);
+    taskID = TezTaskID.getInstance(vertexID, 1);
+    taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
+    credentials = new Credentials();
+
+    amContainerMap = mock(AMContainerMap.class);
+    Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
+
+    eventHandler = mock(EventHandler.class);
+
+    MockClock clock = new MockClock();
+    
+    appContext = mock(AppContext.class);
+    doReturn(eventHandler).when(appContext).getEventHandler();
+    doReturn(dag).when(appContext).getCurrentDAG();
+    doReturn(appAcls).when(appContext).getApplicationACLs();
+    doReturn(amContainerMap).when(appContext).getAllContainers();
+    doReturn(clock).when(appContext).getClock();
+    
+    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
+    doReturn(credentials).when(appContext).getAppCredentials();
+    NodeId nodeId = NodeId.newInstance("localhost", 0);
+    AMContainer amContainer = mock(AMContainer.class);
+    Container container = mock(Container.class);
+    doReturn(nodeId).when(container).getNodeId();
+    doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
+    doReturn(container).when(amContainer).getContainer();
+
+    Configuration conf = new TezConfiguration();
+    UserPayload defaultPayload;
+    try {
+      defaultPayload = TezUtils.createUserPayloadFromConf(conf);
+    } catch (IOException e) {
+      throw new TezUncheckedException(e);
+    }
+    taskAttemptListener = new TaskCommunicatorManagerInterfaceImplForTest(appContext,
+        mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class),
+        Lists.newArrayList(
+            new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+                .setUserPayload(defaultPayload)));
+
+    taskSpec = mock(TaskSpec.class);
+    doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();
+    amContainerTask = new AMContainerTask(taskSpec, null, null, false, 0);
+    containerTask = null;
+  }
+
+  @Test(timeout = 5000)
+  public void testGetTask() throws IOException {
+
+    TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
+    TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
+
+    ContainerId containerId1 = createContainerId(appId, 1);
+    ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
+    containerTask = tezUmbilical.getTask(containerContext1);
+    assertTrue(containerTask.shouldDie());
+
+    ContainerId containerId2 = createContainerId(appId, 2);
+    ContainerContext containerContext2 = new ContainerContext(containerId2.toString());
+    taskAttemptListener.registerRunningContainer(containerId2, 0);
+    containerTask = tezUmbilical.getTask(containerContext2);
+    assertNull(containerTask);
+
+    // Valid task registered
+    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId2, 0);
+    containerTask = tezUmbilical.getTask(containerContext2);
+    assertFalse(containerTask.shouldDie());
+    assertEquals(taskSpec, containerTask.getTaskSpec());
+
+    // Task unregistered. Should respond to heartbeats
+    taskAttemptListener.unregisterTaskAttempt(taskAttemptID, 0, TaskAttemptEndReason.OTHER, null);
+    containerTask = tezUmbilical.getTask(containerContext2);
+    assertNull(containerTask);
+
+    // Container unregistered. Should send a shouldDie = true
+    taskAttemptListener.unregisterRunningContainer(containerId2, 0, ContainerEndReason.OTHER, null);
+    containerTask = tezUmbilical.getTask(containerContext2);
+    assertTrue(containerTask.shouldDie());
+
+    ContainerId containerId3 = createContainerId(appId, 3);
+    ContainerContext containerContext3 = new ContainerContext(containerId3.toString());
+    taskAttemptListener.registerRunningContainer(containerId3, 0);
+
+    // Register task to container3, followed by unregistering container 3 all together
+    TaskSpec taskSpec2 = mock(TaskSpec.class);
+    TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
+    doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
+    AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec, null, null, false, 0);
+    taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId3, 0);
+    taskAttemptListener.unregisterRunningContainer(containerId3, 0, ContainerEndReason.OTHER, null);
+    containerTask = tezUmbilical.getTask(containerContext3);
+    assertTrue(containerTask.shouldDie());
+  }
+
+  @Test(timeout = 5000)
+  public void testGetTaskMultiplePulls() throws IOException {
+    TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
+    TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
+
+    ContainerId containerId1 = createContainerId(appId, 1);
+
+    ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
+    taskAttemptListener.registerRunningContainer(containerId1, 0);
+    containerTask = tezUmbilical.getTask(containerContext1);
+    assertNull(containerTask);
+
+    // Register task
+    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId1, 0);
+    containerTask = tezUmbilical.getTask(containerContext1);
+    assertFalse(containerTask.shouldDie());
+    assertEquals(taskSpec, containerTask.getTaskSpec());
+
+    // Try pulling again - simulates re-use pull
+    containerTask = tezUmbilical.getTask(containerContext1);
+    assertNull(containerTask);
+  }
+
+  @Test (timeout = 5000)
+  public void testTaskEventRouting() throws Exception {
+    List<TezEvent> events =  Arrays.asList(
+      new TezEvent(new TaskStatusUpdateEvent(null, 0.0f, null), null),
+      new TezEvent(DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0])), null),
+      new TezEvent(new TaskAttemptCompletedEvent(), null)
+    );
+
+    generateHeartbeat(events, 0, 1, 0, new ArrayList<TezEvent>());
+
+    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+    verify(eventHandler, times(2)).handle(arg.capture());
+    final List<Event> argAllValues = arg.getAllValues();
+
+    final Event statusUpdateEvent = argAllValues.get(0);
+    assertEquals("First event should be status update", TaskAttemptEventType.TA_STATUS_UPDATE,
+        statusUpdateEvent.getType());
+
+    final Event vertexEvent = argAllValues.get(1);
+    final VertexEventRouteEvent vertexRouteEvent = (VertexEventRouteEvent)vertexEvent;
+    assertEquals("First event should be routed to vertex", VertexEventType.V_ROUTE_EVENT,
+        vertexEvent.getType());
+    assertEquals(EventType.DATA_MOVEMENT_EVENT,
+        vertexRouteEvent.getEvents().get(0).getEventType());
+    assertEquals(EventType.TASK_ATTEMPT_COMPLETED_EVENT,
+        vertexRouteEvent.getEvents().get(1).getEventType());
+
+  }
+
+  @Test (timeout = 5000)
+  public void testTaskEventRoutingTaskAttemptOnly() throws Exception {
+    List<TezEvent> events = Arrays.asList(
+      new TezEvent(new TaskAttemptCompletedEvent(), null)
+    );
+    generateHeartbeat(events, 0, 1, 0, new ArrayList<TezEvent>());
+
+    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+    verify(eventHandler, times(1)).handle(arg.capture());
+    final List<Event> argAllValues = arg.getAllValues();
+
+    final Event event = argAllValues.get(0);
+    assertEquals("only event should be route event", VertexEventType.V_ROUTE_EVENT,
+        event.getType());
+  }
+  
+  @Test (timeout = 5000)
+  public void testTaskHeartbeatResponse() throws Exception {
+    List<TezEvent> events = new ArrayList<TezEvent>();
+    List<TezEvent> eventsToSend = new ArrayList<TezEvent>();
+    TaskHeartbeatResponse response = generateHeartbeat(events, 0, 1, 2, eventsToSend);
+    
+    assertEquals(2, response.getNextFromEventId());
+    assertEquals(eventsToSend, response.getEvents());
+  }
+
+  //try 10 times to allocate random port, fail it if no one is succeed.
+  @Test (timeout = 5000)
+  public void testPortRange() {
+    boolean succeedToAllocate = false;
+    Random rand = new Random();
+    for (int i = 0; i < 10; ++i) {
+      int nextPort = 1024 + rand.nextInt(65535 - 1024);
+      if (testPortRange(nextPort)) {
+        succeedToAllocate = true;
+        break;
+      }
+    }
+    if (!succeedToAllocate) {
+      fail("Can not allocate free port even in 10 iterations for TaskAttemptListener");
+    }
+  }
+
+  // TODO TEZ-2003 Move this into TestTezTaskCommunicator. Potentially other tests as well.
+  @Test (timeout= 5000)
+  public void testPortRange_NotSpecified() throws IOException {
+    Configuration conf = new Configuration();
+    JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(
+        "fakeIdentifier"));
+    Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier,
+        new JobTokenSecretManager());
+    sessionToken.setService(identifier.getJobId());
+    TokenCache.setSessionToken(sessionToken, credentials);
+    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+    taskAttemptListener = new TaskCommunicatorManager(appContext,
+        mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(
+        new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+            .setUserPayload(userPayload)));
+    // no exception happen, should started properly
+    taskAttemptListener.init(conf);
+    taskAttemptListener.start();
+  }
+
+  private boolean testPortRange(int port) {
+    boolean succeedToAllocate = true;
+    try {
+      Configuration conf = new Configuration();
+      
+      JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(
+          "fakeIdentifier"));
+      Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier,
+          new JobTokenSecretManager());
+      sessionToken.setService(identifier.getJobId());
+      TokenCache.setSessionToken(sessionToken, credentials);
+
+      conf.set(TezConfiguration.TEZ_AM_TASK_AM_PORT_RANGE, port + "-" + port);
+      UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+
+      taskAttemptListener = new TaskCommunicatorManager(appContext,
+          mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists
+          .newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+              .setUserPayload(userPayload)));
+      taskAttemptListener.init(conf);
+      taskAttemptListener.start();
+      int resultedPort = taskAttemptListener.getTaskCommunicator(0).getAddress().getPort();
+      assertEquals(port, resultedPort);
+    } catch (Exception e) {
+      succeedToAllocate = false;
+    } finally {
+      if (taskAttemptListener != null) {
+        try {
+          taskAttemptListener.close();
+        } catch (IOException e) {
+          e.printStackTrace();
+          fail("fail to stop TaskAttemptListener");
+        }
+      }
+    }
+    return succeedToAllocate;
+  }
+
+  private TaskHeartbeatResponse generateHeartbeat(List<TezEvent> events,
+      int fromEventId, int maxEvents, int nextFromEventId,
+      List<TezEvent> sendEvents) throws IOException, TezException {
+    ContainerId containerId = createContainerId(appId, 1);
+    Vertex vertex = mock(Vertex.class);
+
+    doReturn(vertex).when(dag).getVertex(vertexID);
+    doReturn("test_vertex").when(vertex).getName();
+    TaskAttemptEventInfo eventInfo = new TaskAttemptEventInfo(nextFromEventId, sendEvents, 0);
+    doReturn(eventInfo).when(vertex).getTaskAttemptTezEvents(taskAttemptID, fromEventId, 0, maxEvents);
+
+    taskAttemptListener.registerRunningContainer(containerId, 0);
+    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId, 0);
+
+    TaskHeartbeatRequest request = mock(TaskHeartbeatRequest.class);
+    doReturn(containerId.toString()).when(request).getContainerIdentifier();
+    doReturn(containerId.toString()).when(request).getContainerIdentifier();
+    doReturn(taskAttemptID).when(request).getTaskAttemptId();
+    doReturn(events).when(request).getEvents();
+    doReturn(maxEvents).when(request).getMaxEvents();
+    doReturn(fromEventId).when(request).getStartIndex();
+
+    return taskAttemptListener.heartbeat(request);
+  }
+
+
+  private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
+    return ContainerId.newInstance(appAttemptId, containerIdx);
+  }
+
+  private static class TaskCommunicatorManagerInterfaceImplForTest extends TaskCommunicatorManager {
+
+    public TaskCommunicatorManagerInterfaceImplForTest(AppContext context,
+                                                       TaskHeartbeatHandler thh,
+                                                       ContainerHeartbeatHandler chh,
+                                                       List<NamedEntityDescriptor> taskCommDescriptors) {
+      super(context, thh, chh, taskCommDescriptors);
+    }
+
+    @Override
+    TaskCommunicator createDefaultTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
+      return new TezTaskCommunicatorImplForTest(taskCommunicatorContext);
+    }
+
+  }
+
+  private static class TezTaskCommunicatorImplForTest extends TezTaskCommunicatorImpl {
+
+    public TezTaskCommunicatorImplForTest(
+        TaskCommunicatorContext taskCommunicatorContext) {
+      super(taskCommunicatorContext);
+    }
+
+    @Override
+    protected void startRpcServer() {
+    }
+
+    @Override
+    protected void stopRpcServer() {
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java
new file mode 100644
index 0000000..d75b0e5
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptKilled;
+import org.apache.tez.dag.app.rm.container.AMContainer;
+import org.apache.tez.dag.app.rm.container.AMContainerMap;
+import org.apache.tez.dag.app.rm.container.AMContainerTask;
+import org.apache.tez.dag.records.TaskAttemptTerminationCause;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+public class TestTaskCommunicatorManager2 {
+
+  @Test(timeout = 5000)
+  public void testTaskAttemptFailedKilled() throws IOException {
+    ApplicationId appId = ApplicationId.newInstance(1000, 1);
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+    Credentials credentials = new Credentials();
+    AppContext appContext = mock(AppContext.class);
+    EventHandler eventHandler = mock(EventHandler.class);
+    DAG dag = mock(DAG.class);
+    AMContainerMap amContainerMap = mock(AMContainerMap.class);
+    Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
+    doReturn(eventHandler).when(appContext).getEventHandler();
+    doReturn(dag).when(appContext).getCurrentDAG();
+    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
+    doReturn(credentials).when(appContext).getAppCredentials();
+    doReturn(appAcls).when(appContext).getApplicationACLs();
+    doReturn(amContainerMap).when(appContext).getAllContainers();
+    NodeId nodeId = NodeId.newInstance("localhost", 0);
+    AMContainer amContainer = mock(AMContainer.class);
+    Container container = mock(Container.class);
+    doReturn(nodeId).when(container).getNodeId();
+    doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
+    doReturn(container).when(amContainer).getContainer();
+
+    Configuration conf = new TezConfiguration();
+    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+    TaskCommunicatorManager taskAttemptListener =
+        new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class),
+            mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(
+            TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload)));
+
+    TaskSpec taskSpec1 = mock(TaskSpec.class);
+    TezTaskAttemptID taskAttemptId1 = mock(TezTaskAttemptID.class);
+    doReturn(taskAttemptId1).when(taskSpec1).getTaskAttemptID();
+    AMContainerTask amContainerTask1 = new AMContainerTask(taskSpec1, null, null, false, 10);
+
+    TaskSpec taskSpec2 = mock(TaskSpec.class);
+    TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
+    doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
+    AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec2, null, null, false, 10);
+
+    ContainerId containerId1 = createContainerId(appId, 1);
+    taskAttemptListener.registerRunningContainer(containerId1, 0);
+    taskAttemptListener.registerTaskAttempt(amContainerTask1, containerId1, 0);
+    ContainerId containerId2 = createContainerId(appId, 2);
+    taskAttemptListener.registerRunningContainer(containerId2, 0);
+    taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId2, 0);
+
+
+    taskAttemptListener
+        .taskFailed(taskAttemptId1, TaskAttemptEndReason.COMMUNICATION_ERROR, "Diagnostics1");
+    taskAttemptListener
+        .taskKilled(taskAttemptId2, TaskAttemptEndReason.EXECUTOR_BUSY, "Diagnostics2");
+
+    ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
+    verify(eventHandler, times(2)).handle(argumentCaptor.capture());
+    assertTrue(argumentCaptor.getAllValues().get(0) instanceof TaskAttemptEventAttemptFailed);
+    assertTrue(argumentCaptor.getAllValues().get(1) instanceof TaskAttemptEventAttemptKilled);
+    TaskAttemptEventAttemptFailed failedEvent =
+        (TaskAttemptEventAttemptFailed) argumentCaptor.getAllValues().get(0);
+    TaskAttemptEventAttemptKilled killedEvent =
+        (TaskAttemptEventAttemptKilled) argumentCaptor.getAllValues().get(1);
+
+    assertEquals("Diagnostics1", failedEvent.getDiagnosticInfo());
+    assertEquals(TaskAttemptTerminationCause.COMMUNICATION_ERROR,
+        failedEvent.getTerminationCause());
+
+    assertEquals("Diagnostics2", killedEvent.getDiagnosticInfo());
+    assertEquals(TaskAttemptTerminationCause.SERVICE_BUSY, killedEvent.getTerminationCause());
+    // TODO TEZ-2003. Verify unregistration from the registered list
+  }
+
+  private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
+    ContainerId containerId = ContainerId.newInstance(appAttemptId, containerIdx);
+    return containerId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java
index 83421a2..f0b89c8 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java
@@ -70,7 +70,7 @@ import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ClusterInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.DAGState;
 import org.apache.tez.dag.app.dag.DAGTerminationCause;
@@ -142,7 +142,7 @@ public class TestCommit {
   private TaskEventDispatcher taskEventDispatcher;
   private VertexEventDispatcher vertexEventDispatcher;
   private DagEventDispatcher dagEventDispatcher;
-  private TaskAttemptListener taskAttemptListener;
+  private TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
   private TaskHeartbeatHandler thh;
   private Clock clock = new SystemClock();
   private DAGFinishEventHandler dagFinishEventHandler;
@@ -317,7 +317,7 @@ public class TestCommit {
     doReturn(historyEventHandler).when(appContext).getHistoryHandler();
     doReturn(aclManager).when(appContext).getAMACLManager();
     dag = new DAGImpl(dagId, conf, dagPlan, dispatcher.getEventHandler(),
-        taskAttemptListener, fsTokens, clock, "user", thh, appContext);
+        taskCommunicatorManagerInterface, fsTokens, clock, "user", thh, appContext);
     doReturn(dag).when(appContext).getCurrentDAG();
     doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();
     ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(8192,10));

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index e268a99..ac4f61b 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -88,7 +88,7 @@ import org.apache.tez.dag.api.records.DAGProtos.TezEntityDescriptorProto;
 import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ClusterInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.DAGScheduler;
 import org.apache.tez.dag.app.dag.DAGState;
@@ -165,7 +165,7 @@ public class TestDAGImpl {
   private TaskEventDispatcher taskEventDispatcher;
   private VertexEventDispatcher vertexEventDispatcher;
   private DagEventDispatcher dagEventDispatcher;
-  private TaskAttemptListener taskAttemptListener;
+  private TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
   private TaskHeartbeatHandler thh;
   private Clock clock = new SystemClock();
   private DAGFinishEventHandler dagFinishEventHandler;
@@ -784,7 +784,7 @@ public class TestDAGImpl {
     doReturn(historyEventHandler).when(appContext).getHistoryHandler();
     doReturn(aclManager).when(appContext).getAMACLManager();
     dag = new DAGImpl(dagId, conf, dagPlan,
-        dispatcher.getEventHandler(),  taskAttemptListener,
+        dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
         fsTokens, clock, "user", thh, appContext);
     dag.entityUpdateTracker = new StateChangeNotifierForTest(dag);
     doReturn(dag).when(appContext).getCurrentDAG();
@@ -795,7 +795,7 @@ public class TestDAGImpl {
     mrrDagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 2);
     mrrDagPlan = createTestMRRDAGPlan();
     mrrDag = new DAGImpl(mrrDagId, conf, mrrDagPlan,
-        dispatcher.getEventHandler(),  taskAttemptListener,
+        dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
         fsTokens, clock, "user", thh,
         mrrAppContext);
     mrrDag.entityUpdateTracker = new StateChangeNotifierForTest(mrrDag);
@@ -811,7 +811,7 @@ public class TestDAGImpl {
     groupDagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 3);
     groupDagPlan = createGroupDAGPlan();
     groupDag = new DAGImpl(groupDagId, conf, groupDagPlan,
-        dispatcher.getEventHandler(),  taskAttemptListener,
+        dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
         fsTokens, clock, "user", thh,
         groupAppContext);
     groupDag.entityUpdateTracker = new StateChangeNotifierForTest(groupDag);
@@ -881,7 +881,7 @@ public class TestDAGImpl {
     dagWithCustomEdgeAppContext = mock(AppContext.class);
     doReturn(aclManager).when(dagWithCustomEdgeAppContext).getAMACLManager();
     dagWithCustomEdge = new DAGImpl(dagWithCustomEdgeId, conf, dagPlanWithCustomEdge,
-        dispatcher.getEventHandler(),  taskAttemptListener,
+        dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
         fsTokens, clock, "user", thh, dagWithCustomEdgeAppContext);
     dagWithCustomEdge.entityUpdateTracker = new StateChangeNotifierForTest(dagWithCustomEdge);
     doReturn(conf).when(dagWithCustomEdgeAppContext).getAMConf();

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
index 792fa63..409c506 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
@@ -37,7 +37,7 @@ import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ClusterInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.DAGState;
 import org.apache.tez.dag.app.dag.DAGTerminationCause;
@@ -91,7 +91,7 @@ public class TestDAGRecovery {
     DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
     dag =
         new DAGImpl(dagId, new Configuration(), dagPlan, mockEventHandler,
-            mock(TaskAttemptListener.class), new Credentials(),
+            mock(TaskCommunicatorManagerInterface.class), new Credentials(),
             new SystemClock(), user, mock(TaskHeartbeatHandler.class),
             mockAppContext);
   }

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index 04bb2df..13c9202 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -64,7 +64,7 @@ import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ClusterInfo;
 import org.apache.tez.dag.app.ContainerContext;
 import org.apache.tez.dag.app.ContainerHeartbeatHandler;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.Task;
 import org.apache.tez.dag.app.dag.TaskAttemptStateInternal;
@@ -145,7 +145,7 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(
         TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1);
     TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
-        mock(TaskAttemptListener.class), new Configuration(), new SystemClock(),
+        mock(TaskCommunicatorManagerInterface.class), new Configuration(), new SystemClock(),
         mock(TaskHeartbeatHandler.class), mock(AppContext.class),
         false, Resource.newInstance(1024, 1), createFakeContainerContext(), false);
 
@@ -180,12 +180,12 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(
         TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1);
     TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
-        mock(TaskAttemptListener.class), new Configuration(), new SystemClock(),
+        mock(TaskCommunicatorManagerInterface.class), new Configuration(), new SystemClock(),
         mock(TaskHeartbeatHandler.class), mock(AppContext.class),
         false, Resource.newInstance(1024, 1), createFakeContainerContext(), false);
 
     TaskAttemptImpl taImplReScheduled = new MockTaskAttemptImpl(taskID, 1, eventHandler,
-        mock(TaskAttemptListener.class), new Configuration(), new SystemClock(),
+        mock(TaskCommunicatorManagerInterface.class), new Configuration(), new SystemClock(),
         mock(TaskHeartbeatHandler.class), mock(AppContext.class),
         true, Resource.newInstance(1024, 1), createFakeContainerContext(), false);
 
@@ -243,7 +243,7 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(
         TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1);
     TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
-        mock(TaskAttemptListener.class), new Configuration(),
+        mock(TaskCommunicatorManagerInterface.class), new Configuration(),
         new SystemClock(), mock(TaskHeartbeatHandler.class),
         mock(AppContext.class), false, Resource.newInstance(1024,
             1), createFakeContainerContext(), false);
@@ -285,7 +285,7 @@ public class TestTaskAttempt {
     TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 0);
 
     MockEventHandler eventHandler = new MockEventHandler();
-    TaskAttemptListener taListener = createMockTaskAttemptListener();
+    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
 
     Configuration taskConf = new Configuration();
     taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -334,7 +334,7 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
 
     MockEventHandler eventHandler = spy(new MockEventHandler());
-    TaskAttemptListener taListener = createMockTaskAttemptListener();
+    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
 
     Configuration taskConf = new Configuration();
     taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -354,7 +354,7 @@ public class TestTaskAttempt {
 
     AppContext appCtx = mock(AppContext.class);
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appCtx);
     containers.addContainerIfNew(container, 0, 0, 0);
 
@@ -434,7 +434,7 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
 
     MockEventHandler eventHandler = new MockEventHandler();
-    TaskAttemptListener taListener = createMockTaskAttemptListener();
+    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
 
     Configuration taskConf = new Configuration();
     taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -454,7 +454,7 @@ public class TestTaskAttempt {
 
     AppContext appCtx = mock(AppContext.class);
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appCtx);
     containers.addContainerIfNew(container, 0, 0, 0);
 
@@ -498,7 +498,7 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
 
     MockEventHandler eventHandler = spy(new MockEventHandler());
-    TaskAttemptListener taListener = createMockTaskAttemptListener();
+    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
 
     Configuration taskConf = new Configuration();
     taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -518,7 +518,7 @@ public class TestTaskAttempt {
 
     AppContext appCtx = mock(AppContext.class);
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appCtx);
     containers.addContainerIfNew(container, 0, 0, 0);
 
@@ -589,7 +589,7 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
 
     MockEventHandler eventHandler = spy(new MockEventHandler());
-    TaskAttemptListener taListener = createMockTaskAttemptListener();
+    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
 
     Configuration taskConf = new Configuration();
     taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -610,7 +610,7 @@ public class TestTaskAttempt {
 
     AppContext appCtx = mock(AppContext.class);
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appCtx);
     containers.addContainerIfNew(container, 0, 0, 0);
 
@@ -719,7 +719,7 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
 
     MockEventHandler eventHandler = spy(new MockEventHandler());
-    TaskAttemptListener taListener = createMockTaskAttemptListener();
+    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
 
     Configuration taskConf = new Configuration();
     taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -740,7 +740,7 @@ public class TestTaskAttempt {
 
     AppContext appCtx = mock(AppContext.class);
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appCtx);
     containers.addContainerIfNew(container, 0, 0, 0);
 
@@ -810,7 +810,7 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
 
     MockEventHandler eventHandler = spy(new MockEventHandler());
-    TaskAttemptListener taListener = createMockTaskAttemptListener();
+    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
 
     Configuration taskConf = new Configuration();
     taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -830,7 +830,7 @@ public class TestTaskAttempt {
 
     AppContext appCtx = mock(AppContext.class);
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appCtx);
     containers.addContainerIfNew(container, 0, 0, 0);
 
@@ -904,7 +904,7 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
 
     MockEventHandler eventHandler = spy(new MockEventHandler());
-    TaskAttemptListener taListener = createMockTaskAttemptListener();
+    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
 
     Configuration taskConf = new Configuration();
     taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -924,7 +924,7 @@ public class TestTaskAttempt {
 
     AppContext appCtx = mock(AppContext.class);
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appCtx);
     containers.addContainerIfNew(container, 0, 0, 0);
 
@@ -1006,7 +1006,7 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
 
     MockEventHandler eventHandler = spy(new MockEventHandler());
-    TaskAttemptListener taListener = createMockTaskAttemptListener();
+    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
 
     Configuration taskConf = new Configuration();
     taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -1026,7 +1026,7 @@ public class TestTaskAttempt {
 
     AppContext appCtx = mock(AppContext.class);
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appCtx);
     containers.addContainerIfNew(container, 0, 0, 0);
 
@@ -1105,7 +1105,7 @@ public class TestTaskAttempt {
 
     MockEventHandler mockEh = new MockEventHandler();
     MockEventHandler eventHandler = spy(mockEh);
-    TaskAttemptListener taListener = createMockTaskAttemptListener();
+    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
 
     Configuration taskConf = new Configuration();
     taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -1125,7 +1125,7 @@ public class TestTaskAttempt {
 
     AppContext appCtx = mock(AppContext.class);
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appCtx);
     containers.addContainerIfNew(container, 0, 0, 0);
 
@@ -1249,7 +1249,7 @@ public class TestTaskAttempt {
     TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
 
     MockEventHandler eventHandler = spy(new MockEventHandler());
-    TaskAttemptListener taListener = createMockTaskAttemptListener();
+    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
 
     Configuration taskConf = new Configuration();
     taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -1268,7 +1268,7 @@ public class TestTaskAttempt {
 
     AppContext appCtx = mock(AppContext.class);
     AMContainerMap containers = new AMContainerMap(
-        mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+        mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
         new ContainerContextMatcher(), appCtx);
     containers.addContainerIfNew(container, 0, 0, 0);
 
@@ -1324,7 +1324,7 @@ public class TestTaskAttempt {
     public int taskAttemptStartedEventLogged = 0;
     public int taskAttemptFinishedEventLogged = 0;
     public MockTaskAttemptImpl(TezTaskID taskId, int attemptNumber,
-        EventHandler eventHandler, TaskAttemptListener tal,
+        EventHandler eventHandler, TaskCommunicatorManagerInterface tal,
         Configuration conf, Clock clock,
         TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
         boolean isRescheduled,
@@ -1378,8 +1378,8 @@ public class TestTaskAttempt {
         new Credentials(), new HashMap<String, String>(), "");
   }
 
-  private TaskAttemptListener createMockTaskAttemptListener() {
-    TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+  private TaskCommunicatorManagerInterface createMockTaskAttemptListener() {
+    TaskCommunicatorManagerInterface taListener = mock(TaskCommunicatorManagerInterface.class);
     TaskCommunicator taskComm = mock(TaskCommunicator.class);
     doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
     doReturn(taskComm).when(taListener).getTaskCommunicator(0);

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
index 4a797e0..6bbfc3d 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
@@ -42,7 +42,7 @@ import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.Task;
 import org.apache.tez.dag.app.dag.TaskAttemptStateInternal;
@@ -146,7 +146,7 @@ public class TestTaskAttemptRecovery {
         TezTaskID.fromString("task_1407371892933_0001_1_00_000000");
     ta =
         new TaskAttemptImpl(taskId, 0, mockEventHandler,
-            mock(TaskAttemptListener.class), new Configuration(),
+            mock(TaskCommunicatorManagerInterface.class), new Configuration(),
             new SystemClock(), mock(TaskHeartbeatHandler.class),
             mockAppContext, false, Resource.newInstance(1, 1),
             mock(ContainerContext.class), false, mockTask);

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 807f277..24c9664 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -50,7 +50,7 @@ import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.StateChangeNotifier;
 import org.apache.tez.dag.app.dag.TaskStateInternal;
@@ -85,7 +85,7 @@ public class TestTaskImpl {
   private final int partition = 1;
 
   private Configuration conf;
-  private TaskAttemptListener taskAttemptListener;
+  private TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
   private TaskHeartbeatHandler taskHeartbeatHandler;
   private Credentials credentials;
   private Clock clock;
@@ -122,7 +122,7 @@ public class TestTaskImpl {
   @Before
   public void setup() {
     conf = new Configuration();
-    taskAttemptListener = mock(TaskAttemptListener.class);
+    taskCommunicatorManagerInterface = mock(TaskCommunicatorManagerInterface.class);
     taskHeartbeatHandler = mock(TaskHeartbeatHandler.class);
     credentials = new Credentials();
     clock = new SystemClock();
@@ -151,7 +151,7 @@ public class TestTaskImpl {
     eventHandler = new TestEventHandler();
     
     mockTask = new MockTaskImpl(vertexId, partition,
-        eventHandler, conf, taskAttemptListener, clock,
+        eventHandler, conf, taskCommunicatorManagerInterface, clock,
         taskHeartbeatHandler, appContext, leafVertex,
         taskResource, containerContext, vertex);
     mockTaskSpec = mock(TaskSpec.class);
@@ -698,11 +698,11 @@ public class TestTaskImpl {
 
     public MockTaskImpl(TezVertexID vertexId, int partition,
         EventHandler eventHandler, Configuration conf,
-        TaskAttemptListener taskAttemptListener, Clock clock,
+        TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Clock clock,
         TaskHeartbeatHandler thh, AppContext appContext, boolean leafVertex,
         Resource resource,
         ContainerContext containerContext, Vertex vertex) {
-      super(vertexId, partition, eventHandler, conf, taskAttemptListener,
+      super(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface,
           clock, thh, appContext, leafVertex, resource,
           containerContext, mock(StateChangeNotifier.class), vertex);
       this.vertex = vertex;
@@ -711,7 +711,7 @@ public class TestTaskImpl {
     @Override
     protected TaskAttemptImpl createAttempt(int attemptNumber, TezTaskAttemptID schedCausalTA) {
       MockTaskAttemptImpl attempt = new MockTaskAttemptImpl(getTaskId(),
-          attemptNumber, eventHandler, taskAttemptListener,
+          attemptNumber, eventHandler, taskCommunicatorManagerInterface,
           conf, clock, taskHeartbeatHandler, appContext,
           true, taskResource, containerContext, schedCausalTA);
       taskAttempts.add(attempt);
@@ -757,7 +757,7 @@ public class TestTaskImpl {
     private TaskAttemptState state = TaskAttemptState.NEW;
 
     public MockTaskAttemptImpl(TezTaskID taskId, int attemptNumber,
-        EventHandler eventHandler, TaskAttemptListener tal, Configuration conf,
+        EventHandler eventHandler, TaskCommunicatorManagerInterface tal, Configuration conf,
         Clock clock, TaskHeartbeatHandler thh, AppContext appContext,
         boolean isRescheduled,
         Resource resource, ContainerContext containerContext, TezTaskAttemptID schedCausalTA) {

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
index 1d22e06..eca8274 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
@@ -46,7 +46,7 @@ import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.StateChangeNotifier;
 import org.apache.tez.dag.app.dag.TaskAttemptStateInternal;
@@ -189,7 +189,7 @@ public class TestTaskRecovery {
     when(mockAppContext.getHistoryHandler()).thenReturn(mockHistoryEventHandler);
     task =
         new TaskImpl(vertexId, 0, dispatcher.getEventHandler(),
-            new Configuration(), mock(TaskAttemptListener.class),
+            new Configuration(), mock(TaskCommunicatorManagerInterface.class),
             new SystemClock(), mock(TaskHeartbeatHandler.class),
             mockAppContext, false, Resource.newInstance(1, 1),
             mock(ContainerContext.class), mock(StateChangeNotifier.class), vertex);


[7/7] tez git commit: TEZ-2708. Rename classes and variables post TEZ-2003 changes. (sseth)

Posted by ss...@apache.org.
TEZ-2708. Rename classes and variables post TEZ-2003 changes. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/8b278ea8
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/8b278ea8
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/8b278ea8

Branch: refs/heads/master
Commit: 8b278ea84f4c64e7144c07fa79b38a6a719541d2
Parents: dc0ee01
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Aug 25 16:48:00 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Aug 25 16:48:00 2015 -0700

----------------------------------------------------------------------
 tez-dag/findbugs-exclude.xml                    |  15 +-
 .../java/org/apache/tez/dag/app/AppContext.java |   4 +-
 .../dag/app/ContainerLauncherContextImpl.java   |   4 +-
 .../org/apache/tez/dag/app/DAGAppMaster.java    | 113 +--
 .../apache/tez/dag/app/TaskAttemptListener.java |  46 --
 .../dag/app/TaskAttemptListenerImpTezDag.java   | 449 -----------
 .../dag/app/TaskCommunicatorContextImpl.java    |  22 +-
 .../tez/dag/app/TaskCommunicatorManager.java    | 449 +++++++++++
 .../app/TaskCommunicatorManagerInterface.java   |  46 ++
 .../apache/tez/dag/app/dag/impl/DAGImpl.java    |  10 +-
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java   |   8 +-
 .../apache/tez/dag/app/dag/impl/TaskImpl.java   |  10 +-
 .../apache/tez/dag/app/dag/impl/VertexImpl.java |  10 +-
 .../dag/app/launcher/ContainerLauncherImpl.java | 410 ----------
 .../app/launcher/ContainerLauncherManager.java  | 217 +++++
 .../app/launcher/ContainerLauncherRouter.java   | 215 -----
 .../app/launcher/LocalContainerLauncher.java    |   8 +-
 .../app/launcher/TezContainerLauncherImpl.java  | 410 ++++++++++
 .../tez/dag/app/rm/ContainerLauncherEvent.java  | 116 +++
 .../dag/app/rm/ContainerLauncherEventType.java  |  25 +
 .../rm/ContainerLauncherLaunchRequestEvent.java |  79 ++
 .../rm/ContainerLauncherStopRequestEvent.java   |  34 +
 .../tez/dag/app/rm/NMCommunicatorEvent.java     | 115 ---
 .../tez/dag/app/rm/NMCommunicatorEventType.java |  25 -
 .../rm/NMCommunicatorLaunchRequestEvent.java    |  78 --
 .../app/rm/NMCommunicatorStopRequestEvent.java  |  33 -
 .../dag/app/rm/TaskSchedulerContextImpl.java    |  29 +-
 .../dag/app/rm/TaskSchedulerEventHandler.java   | 784 ------------------
 .../tez/dag/app/rm/TaskSchedulerManager.java    | 786 +++++++++++++++++++
 .../dag/app/rm/container/AMContainerImpl.java   |  26 +-
 .../dag/app/rm/container/AMContainerMap.java    |   6 +-
 .../apache/tez/dag/app/MockDAGAppMaster.java    |  23 +-
 .../tez/dag/app/TestMockDAGAppMaster.java       |   2 +-
 .../app/TestTaskAttemptListenerImplTezDag.java  | 426 ----------
 .../app/TestTaskAttemptListenerImplTezDag2.java | 137 ----
 .../app/TestTaskCommunicatorContextImpl.java    |   2 +-
 .../dag/app/TestTaskCommunicatorManager.java    |   2 +-
 .../dag/app/TestTaskCommunicatorManager1.java   | 425 ++++++++++
 .../dag/app/TestTaskCommunicatorManager2.java   | 136 ++++
 .../apache/tez/dag/app/dag/impl/TestCommit.java |   6 +-
 .../tez/dag/app/dag/impl/TestDAGImpl.java       |  12 +-
 .../tez/dag/app/dag/impl/TestDAGRecovery.java   |   4 +-
 .../tez/dag/app/dag/impl/TestTaskAttempt.java   |  58 +-
 .../app/dag/impl/TestTaskAttemptRecovery.java   |   4 +-
 .../tez/dag/app/dag/impl/TestTaskImpl.java      |  16 +-
 .../tez/dag/app/dag/impl/TestTaskRecovery.java  |   4 +-
 .../tez/dag/app/dag/impl/TestVertexImpl.java    |  31 +-
 .../tez/dag/app/dag/impl/TestVertexImpl2.java   |   4 +-
 .../dag/app/dag/impl/TestVertexRecovery.java    |  10 +-
 .../launcher/TestContainerLauncherManager.java  | 359 +++++++++
 .../launcher/TestContainerLauncherRouter.java   | 359 ---------
 .../tez/dag/app/rm/TestContainerReuse.java      | 310 ++++----
 .../app/rm/TestTaskSchedulerEventHandler.java   | 707 -----------------
 .../dag/app/rm/TestTaskSchedulerHelpers.java    |  14 +-
 .../dag/app/rm/TestTaskSchedulerManager.java    | 708 +++++++++++++++++
 .../dag/app/rm/container/TestAMContainer.java   |  24 +-
 .../app/rm/container/TestAMContainerMap.java    |   6 +-
 .../tez/dag/app/rm/node/TestAMNodeTracker.java  |  38 +-
 .../tez/service/impl/ContainerRunnerImpl.java   |  23 +-
 .../apache/tez/runtime/task/TezTaskRunner.java  | 451 -----------
 .../tez/runtime/task/TestTaskExecution.java     | 362 ---------
 .../tez/runtime/task/TestTaskExecution2.java    |   2 +-
 62 files changed, 4220 insertions(+), 5027 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/tez-dag/findbugs-exclude.xml b/tez-dag/findbugs-exclude.xml
index 9d15035..6db3b7c 100644
--- a/tez-dag/findbugs-exclude.xml
+++ b/tez-dag/findbugs-exclude.xml
@@ -87,13 +87,13 @@
   </Match>
 
   <Match>
-    <Class name="~org\.apache\.tez\.dag\.app\.rm\.TaskSchedulerEventHandler"/>
+    <Class name="~org\.apache\.tez\.dag\.app\.rm\.TaskSchedulerManager"/>
     <Bug pattern="BC_UNCONFIRMED_CAST"/>
   </Match>
 
   <Match>
-    <Class name="org.apache.tez.dag.app.launcher.ContainerLauncherRouter" />
-    <Method name="handle" params="org.apache.tez.dag.app.rm.NMCommunicatorEvent" returns="void" />
+    <Class name="org.apache.tez.dag.app.launcher.ContainerLauncherManager" />
+    <Method name="handle" params="org.apache.tez.dag.app.rm.ContainerLauncherEvent" returns="void" />
     <Bug pattern="BC_UNCONFIRMED_CAST" />
   </Match>
 
@@ -151,7 +151,7 @@
       <Field name="context"/>
       <Field name="currentDAG"/>
       <Field name="state"/>
-      <Field name="taskSchedulerEventHandler"/>
+      <Field name="taskSchedulerManager"/>
       <Field name="versionMismatch"/>
       <Field name="versionMismatchDiagnostics"/>
       <Field name="containers"/>
@@ -165,13 +165,6 @@
     <Bug pattern="IS2_INCONSISTENT_SYNC"/>
   </Match>
 
-  <!-- TEZ-1955 -->
-  <Match>
-    <Class name="org.apache.tez.dag.app.rm.TaskSchedulerEventHandler"/>
-    <Field name="taskScheduler"/>
-    <Bug pattern="IS2_INCONSISTENT_SYNC"/>
-  </Match>
-
   <!-- TEZ-1956 -->
   <Match>
     <Class name="org.apache.tez.dag.app.rm.YarnTaskSchedulerService"/>

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
index 516fcef..657267b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.app.rm.TaskSchedulerEventHandler;
+import org.apache.tez.dag.app.rm.TaskSchedulerManager;
 import org.apache.tez.dag.app.rm.container.AMContainerMap;
 import org.apache.tez.dag.app.rm.node.AMNodeTracker;
 import org.apache.tez.common.security.ACLManager;
@@ -88,7 +88,7 @@ public interface AppContext {
 
   AMNodeTracker getNodeTracker();
 
-  TaskSchedulerEventHandler getTaskScheduler();
+  TaskSchedulerManager getTaskScheduler();
 
   boolean isSession();
 

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java
index 3a2efc5..20bfd13 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java
@@ -33,10 +33,10 @@ import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
 public class ContainerLauncherContextImpl implements ContainerLauncherContext {
 
   private final AppContext context;
-  private final TaskAttemptListener tal;
+  private final TaskCommunicatorManagerInterface tal;
   private final UserPayload initialUserPayload;
 
-  public ContainerLauncherContextImpl(AppContext appContext, TaskAttemptListener tal, UserPayload initialUserPayload) {
+  public ContainerLauncherContextImpl(AppContext appContext, TaskCommunicatorManagerInterface tal, UserPayload initialUserPayload) {
     this.context = appContext;
     this.tal = tal;
     this.initialUserPayload = initialUserPayload;

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 84b3095..34e7c2a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -40,7 +40,6 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -149,10 +148,10 @@ import org.apache.tez.dag.app.dag.event.TaskEventType;
 import org.apache.tez.dag.app.dag.event.VertexEvent;
 import org.apache.tez.dag.app.dag.event.VertexEventType;
 import org.apache.tez.dag.app.dag.impl.DAGImpl;
-import org.apache.tez.dag.app.launcher.ContainerLauncherRouter;
+import org.apache.tez.dag.app.launcher.ContainerLauncherManager;
 import org.apache.tez.dag.app.rm.AMSchedulerEventType;
-import org.apache.tez.dag.app.rm.NMCommunicatorEventType;
-import org.apache.tez.dag.app.rm.TaskSchedulerEventHandler;
+import org.apache.tez.dag.app.rm.ContainerLauncherEventType;
+import org.apache.tez.dag.app.rm.TaskSchedulerManager;
 import org.apache.tez.dag.app.rm.container.AMContainerEventType;
 import org.apache.tez.dag.app.rm.container.AMContainerMap;
 import org.apache.tez.dag.app.rm.container.ContainerContextMatcher;
@@ -236,16 +235,16 @@ public class DAGAppMaster extends AbstractService {
   private AppContext context;
   private Configuration amConf;
   private AsyncDispatcher dispatcher;
-  private ContainerLauncherRouter containerLauncherRouter;
+  private ContainerLauncherManager containerLauncherManager;
   private ContainerHeartbeatHandler containerHeartbeatHandler;
   private TaskHeartbeatHandler taskHeartbeatHandler;
-  private TaskAttemptListener taskAttemptListener;
+  private TaskCommunicatorManagerInterface taskCommunicatorManager;
   private JobTokenSecretManager jobTokenSecretManager =
       new JobTokenSecretManager();
   private Token<JobTokenIdentifier> sessionToken;
   private DagEventDispatcher dagEventDispatcher;
   private VertexEventDispatcher vertexEventDispatcher;
-  private TaskSchedulerEventHandler taskSchedulerEventHandler;
+  private TaskSchedulerManager taskSchedulerManager;
   private WebUIService webUIService;
   private HistoryEventHandler historyEventHandler;
   private final Map<String, LocalResource> amResources = new HashMap<String, LocalResource>();
@@ -472,13 +471,13 @@ public class DAGAppMaster extends AbstractService {
 
 
     //service to handle requests to TaskUmbilicalProtocol
-    taskAttemptListener = createTaskAttemptListener(context,
+    taskCommunicatorManager = createTaskCommunicatorManager(context,
         taskHeartbeatHandler, containerHeartbeatHandler, taskCommunicatorDescriptors);
-    addIfService(taskAttemptListener, true);
+    addIfService(taskCommunicatorManager, true);
 
     containerSignatureMatcher = createContainerSignatureMatcher();
     containers = new AMContainerMap(containerHeartbeatHandler,
-        taskAttemptListener, containerSignatureMatcher, context);
+        taskCommunicatorManager, containerSignatureMatcher, context);
     addIfService(containers, true);
     dispatcher.register(AMContainerEventType.class, containers);
 
@@ -521,29 +520,30 @@ public class DAGAppMaster extends AbstractService {
 
 
 
-    this.taskSchedulerEventHandler = new TaskSchedulerEventHandler(context,
+    this.taskSchedulerManager = new TaskSchedulerManager(context,
         clientRpcServer, dispatcher.getEventHandler(), containerSignatureMatcher, webUIService,
         taskSchedulerDescriptors, isLocal);
-    addIfService(taskSchedulerEventHandler, true);
+    addIfService(taskSchedulerManager, true);
 
     if (enableWebUIService()) {
-      addIfServiceDependency(taskSchedulerEventHandler, webUIService);
+      addIfServiceDependency(taskSchedulerManager, webUIService);
     }
 
     if (isLastAMRetry) {
       LOG.info("AM will unregister as this is the last attempt"
           + ", currentAttempt=" + appAttemptID.getAttemptId()
           + ", maxAttempts=" + maxAppAttempts);
-      this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+      this.taskSchedulerManager.setShouldUnregisterFlag();
     }
 
     dispatcher.register(AMSchedulerEventType.class,
-        taskSchedulerEventHandler);
-    addIfServiceDependency(taskSchedulerEventHandler, clientRpcServer);
+        taskSchedulerManager);
+    addIfServiceDependency(taskSchedulerManager, clientRpcServer);
 
-    this.containerLauncherRouter = createContainerLauncherRouter(containerLauncherDescriptors, isLocal);
-    addIfService(containerLauncherRouter, true);
-    dispatcher.register(NMCommunicatorEventType.class, containerLauncherRouter);
+    this.containerLauncherManager = createContainerLauncherManager(containerLauncherDescriptors,
+        isLocal);
+    addIfService(containerLauncherManager, true);
+    dispatcher.register(ContainerLauncherEventType.class, containerLauncherManager);
 
     historyEventHandler = createHistoryEventHandler(context);
     addIfService(historyEventHandler, true);
@@ -632,8 +632,8 @@ public class DAGAppMaster extends AbstractService {
   }
   
   @VisibleForTesting
-  protected TaskSchedulerEventHandler getTaskSchedulerEventHandler() {
-    return taskSchedulerEventHandler;
+  protected TaskSchedulerManager getTaskSchedulerManager() {
+    return taskSchedulerManager;
   }
 
   @VisibleForTesting
@@ -661,7 +661,7 @@ public class DAGAppMaster extends AbstractService {
         sendEvent(new DAGEvent(currentDAG.getID(), DAGEventType.INTERNAL_ERROR));
       } else {
         LOG.info("Internal Error. Finishing directly as no dag is active.");
-        this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+        this.taskSchedulerManager.setShouldUnregisterFlag();
         shutdownHandler.shutdown();
       }
       break;
@@ -673,7 +673,7 @@ public class DAGAppMaster extends AbstractService {
       System.out.println(timeStamp + " Completed Dag: " + finishEvt.getDAGId().toString());
       if (!isSession) {
         LOG.info("Not a session, AM will unregister as DAG has completed");
-        this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+        this.taskSchedulerManager.setShouldUnregisterFlag();
         _updateLoggers(currentDAG, "_post");
         setStateOnDAGCompletion();
         LOG.info("Shutting down on completion of dag:" +
@@ -722,7 +722,7 @@ public class DAGAppMaster extends AbstractService {
               + finishEvt.getDAGState()
               + ". Error. Shutting down.");
           state = DAGAppMasterState.ERROR;
-          this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+          this.taskSchedulerManager.setShouldUnregisterFlag();
           shutdownHandler.shutdown();
           break;
         }
@@ -741,11 +741,11 @@ public class DAGAppMaster extends AbstractService {
 
             // Leaving the taskSchedulerEventHandler here for now. Doesn't generate new events.
             // However, eventually it needs to be moved out.
-            this.taskSchedulerEventHandler.dagCompleted();
+            this.taskSchedulerManager.dagCompleted();
             state = DAGAppMasterState.IDLE;
           } else {
             LOG.info("Session shutting down now.");
-            this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+            this.taskSchedulerManager.setShouldUnregisterFlag();
             if (this.historyEventHandler.hasRecoveryFailed()) {
               state = DAGAppMasterState.FAILED;
             } else {
@@ -772,8 +772,8 @@ public class DAGAppMaster extends AbstractService {
       DAGAppMasterEventDagCleanup cleanupEvent = (DAGAppMasterEventDagCleanup) event;
       LOG.info("Cleaning up DAG: name=" + cleanupEvent.getDag().getName() + ", with id=" +
           cleanupEvent.getDag().getID());
-      containerLauncherRouter.dagComplete(cleanupEvent.getDag());
-      taskAttemptListener.dagComplete(cleanupEvent.getDag());
+      containerLauncherManager.dagComplete(cleanupEvent.getDag());
+      taskCommunicatorManager.dagComplete(cleanupEvent.getDag());
       nodes.dagComplete(cleanupEvent.getDag());
       containers.dagComplete(cleanupEvent.getDag());
       TezTaskAttemptID.clearCache();
@@ -785,9 +785,9 @@ public class DAGAppMaster extends AbstractService {
       break;
     case NEW_DAG_SUBMITTED:
       // Inform sub-components that a new DAG has been submitted.
-      taskSchedulerEventHandler.dagSubmitted();
-      containerLauncherRouter.dagSubmitted();
-      taskAttemptListener.dagSubmitted();
+      taskSchedulerManager.dagSubmitted();
+      containerLauncherManager.dagSubmitted();
+      taskCommunicatorManager.dagSubmitted();
       break;
     default:
       throw new TezUncheckedException(
@@ -917,7 +917,7 @@ public class DAGAppMaster extends AbstractService {
     // create single dag
     DAGImpl newDag =
         new DAGImpl(dagId, amConf, dagPB, dispatcher.getEventHandler(),
-            taskAttemptListener, dagCredentials, clock,
+            taskCommunicatorManager, dagCredentials, clock,
             appMasterUgi.getShortUserName(),
             taskHeartbeatHandler, context);
 
@@ -1048,13 +1048,13 @@ public class DAGAppMaster extends AbstractService {
     }
   }
 
-  protected TaskAttemptListener createTaskAttemptListener(AppContext context,
-                                                          TaskHeartbeatHandler thh,
-                                                          ContainerHeartbeatHandler chh,
-                                                          List<NamedEntityDescriptor> entityDescriptors) {
-    TaskAttemptListener lis =
-        new TaskAttemptListenerImpTezDag(context, thh, chh, entityDescriptors);
-    return lis;
+  protected TaskCommunicatorManagerInterface createTaskCommunicatorManager(AppContext context,
+                                                                           TaskHeartbeatHandler thh,
+                                                                           ContainerHeartbeatHandler chh,
+                                                                           List<NamedEntityDescriptor> entityDescriptors) {
+    TaskCommunicatorManagerInterface tcm =
+        new TaskCommunicatorManager(context, thh, chh, entityDescriptors);
+    return tcm;
   }
 
   protected TaskHeartbeatHandler createTaskHeartbeatHandler(AppContext context,
@@ -1074,10 +1074,11 @@ public class DAGAppMaster extends AbstractService {
     return chh;
   }
 
-  protected ContainerLauncherRouter createContainerLauncherRouter(List<NamedEntityDescriptor> containerLauncherDescriptors,
-                                                                  boolean isLocal) throws
+  protected ContainerLauncherManager createContainerLauncherManager(
+      List<NamedEntityDescriptor> containerLauncherDescriptors,
+      boolean isLocal) throws
       UnknownHostException {
-    return new ContainerLauncherRouter(context, taskAttemptListener, workingDirectory,
+    return new ContainerLauncherManager(context, taskCommunicatorManager, workingDirectory,
         containerLauncherDescriptors, isLocal);
   }
 
@@ -1101,12 +1102,12 @@ public class DAGAppMaster extends AbstractService {
     return dispatcher;
   }
 
-  public ContainerLauncherRouter getContainerLauncherRouter() {
-    return containerLauncherRouter;
+  public ContainerLauncherManager getContainerLauncherManager() {
+    return containerLauncherManager;
   }
 
-  public TaskAttemptListener getTaskAttemptListener() {
-    return taskAttemptListener;
+  public TaskCommunicatorManagerInterface getTaskCommunicatorManager() {
+    return taskCommunicatorManager;
   }
 
   public ContainerId getAppContainerId() {
@@ -1213,7 +1214,7 @@ public class DAGAppMaster extends AbstractService {
   public void shutdownTezAM() throws TezException {
     sessionStopped.set(true);
     synchronized (this) {
-      this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+      this.taskSchedulerManager.setShouldUnregisterFlag();
       if (currentDAG != null
           && !currentDAG.isComplete()) {
         //send a DAG_KILL message
@@ -1473,8 +1474,8 @@ public class DAGAppMaster extends AbstractService {
     }
 
     @Override
-    public TaskSchedulerEventHandler getTaskScheduler() {
-      return taskSchedulerEventHandler;
+    public TaskSchedulerManager getTaskScheduler() {
+      return taskSchedulerManager;
     }
 
     @Override
@@ -1574,7 +1575,7 @@ public class DAGAppMaster extends AbstractService {
         throw new TezUncheckedException(
             "Cannot get ApplicationACLs before all services have started");
       }
-      return taskSchedulerEventHandler.getApplicationAcls();
+      return taskSchedulerManager.getApplicationAcls();
     }
 
     @Override
@@ -1805,7 +1806,7 @@ public class DAGAppMaster extends AbstractService {
 
     if (versionMismatch) {
       // Short-circuit and return as no DAG should not be run
-      this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+      this.taskSchedulerManager.setShouldUnregisterFlag();
       shutdownHandler.shutdown();
       return;
     }
@@ -1825,7 +1826,7 @@ public class DAGAppMaster extends AbstractService {
       LOG.error("Error occurred when trying to recover data from previous attempt."
           + " Shutting down AM", e);
       this.state = DAGAppMasterState.ERROR;
-      this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+      this.taskSchedulerManager.setShouldUnregisterFlag();
       shutdownHandler.shutdown();
       return;
     }
@@ -1929,7 +1930,7 @@ public class DAGAppMaster extends AbstractService {
 
 
   private void initiateStop() {
-    taskSchedulerEventHandler.initiateStop();
+    taskSchedulerManager.initiateStop();
   }
 
   @Override
@@ -1954,8 +1955,8 @@ public class DAGAppMaster extends AbstractService {
         LOG.debug("Checking whether tez scratch data dir should be deleted, deleteTezScratchData="
             + deleteTezScratchData);
       }
-      if (deleteTezScratchData && this.taskSchedulerEventHandler != null
-          && this.taskSchedulerEventHandler.hasUnregistered()) {
+      if (deleteTezScratchData && this.taskSchedulerManager != null
+          && this.taskSchedulerManager.hasUnregistered()) {
         // Delete tez scratch data dir
         if (this.tezSystemStagingDir != null) {
           try {
@@ -2208,7 +2209,7 @@ public class DAGAppMaster extends AbstractService {
         // Notify TaskScheduler that a SIGTERM has been received so that it
         // unregisters quickly with proper status
         LOG.info("DAGAppMaster received a signal. Signaling TaskScheduler");
-        appMaster.taskSchedulerEventHandler.setSignalled(true);
+        appMaster.taskSchedulerManager.setSignalled(true);
       }
 
       if (EnumSet.of(DAGAppMasterState.NEW, DAGAppMasterState.INITED,

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
deleted file mode 100644
index 761bdb0..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.serviceplugins.api.ContainerEndReason;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.api.TaskCommunicator;
-import org.apache.tez.dag.app.rm.container.AMContainerTask;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-/**
- * This class listens for changes to the state of a Task.
- */
-public interface TaskAttemptListener {
-
-  void registerRunningContainer(ContainerId containerId, int taskCommId);
-
-  void registerTaskAttempt(AMContainerTask amContainerTask, ContainerId containerId, int taskCommId);
-  
-  void unregisterRunningContainer(ContainerId containerId, int taskCommId, ContainerEndReason endReason, String diagnostics);
-  
-  void unregisterTaskAttempt(TezTaskAttemptID attemptID, int taskCommId, TaskAttemptEndReason endReason, String diagnostics);
-
-  void dagComplete(DAG dag);
-
-  void dagSubmitted();
-
-  TaskCommunicator getTaskCommunicator(int taskCommIndex);
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
deleted file mode 100644
index 185193f..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ /dev/null
@@ -1,449 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.tez.dag.app;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.commons.collections4.ListUtils;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
-import org.apache.tez.serviceplugins.api.ContainerEndReason;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEvent;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate;
-import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent;
-import org.apache.tez.runtime.api.impl.EventType;
-import org.apache.tez.dag.api.event.VertexStateUpdate;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.tez.common.ReflectionUtils;
-import org.apache.tez.common.TezUtilsInternal;
-import org.apache.tez.dag.api.TaskCommunicator;
-import org.apache.tez.dag.api.TaskCommunicatorContext;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.dag.api.TaskHeartbeatResponse;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.TaskHeartbeatRequest;
-import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.app.dag.Task;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptKilled;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
-import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
-import org.apache.tez.dag.app.rm.container.AMContainerTask;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezVertexID;
-import org.apache.tez.runtime.api.impl.TezEvent;
-
-
-@SuppressWarnings("unchecked")
-@InterfaceAudience.Private
-public class TaskAttemptListenerImpTezDag extends AbstractService implements
-    TaskAttemptListener {
-
-  private static final Logger LOG = LoggerFactory
-      .getLogger(TaskAttemptListenerImpTezDag.class);
-
-  private final AppContext context;
-  private final TaskCommunicator[] taskCommunicators;
-  private final TaskCommunicatorContext[] taskCommunicatorContexts;
-  protected final ServicePluginLifecycleAbstractService []taskCommunicatorServiceWrappers;
-
-  protected final TaskHeartbeatHandler taskHeartbeatHandler;
-  protected final ContainerHeartbeatHandler containerHeartbeatHandler;
-
-  private final TaskHeartbeatResponse RESPONSE_SHOULD_DIE = new TaskHeartbeatResponse(true, null, 0, 0);
-
-  private final ConcurrentMap<TezTaskAttemptID, ContainerId> registeredAttempts =
-      new ConcurrentHashMap<TezTaskAttemptID, ContainerId>();
-  private final ConcurrentMap<ContainerId, ContainerInfo> registeredContainers =
-      new ConcurrentHashMap<ContainerId, ContainerInfo>();
-
-  // Defined primarily to work around ConcurrentMaps not accepting null values
-  private static final class ContainerInfo {
-    TezTaskAttemptID taskAttemptId;
-    ContainerInfo(TezTaskAttemptID taskAttemptId) {
-      this.taskAttemptId = taskAttemptId;
-    }
-  }
-
-  private static final ContainerInfo NULL_CONTAINER_INFO = new ContainerInfo(null);
-
-
-  public TaskAttemptListenerImpTezDag(AppContext context,
-                                      TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
-                                      List<NamedEntityDescriptor> taskCommunicatorDescriptors) {
-    super(TaskAttemptListenerImpTezDag.class.getName());
-    this.context = context;
-    this.taskHeartbeatHandler = thh;
-    this.containerHeartbeatHandler = chh;
-    Preconditions.checkArgument(
-        taskCommunicatorDescriptors != null && !taskCommunicatorDescriptors.isEmpty(),
-        "TaskCommunicators must be specified");
-    this.taskCommunicators = new TaskCommunicator[taskCommunicatorDescriptors.size()];
-    this.taskCommunicatorContexts = new TaskCommunicatorContext[taskCommunicatorDescriptors.size()];
-    this.taskCommunicatorServiceWrappers = new ServicePluginLifecycleAbstractService[taskCommunicatorDescriptors.size()];
-    for (int i = 0 ; i < taskCommunicatorDescriptors.size() ; i++) {
-      UserPayload userPayload = taskCommunicatorDescriptors.get(i).getUserPayload();
-      taskCommunicatorContexts[i] = new TaskCommunicatorContextImpl(context, this, userPayload, i);
-      taskCommunicators[i] = createTaskCommunicator(taskCommunicatorDescriptors.get(i), i);
-      taskCommunicatorServiceWrappers[i] = new ServicePluginLifecycleAbstractService(taskCommunicators[i]);
-    }
-    // TODO TEZ-2118 Start using taskCommunicator indices properly
-  }
-
-  @Override
-  public void serviceStart() {
-    // TODO Why is init tied to serviceStart
-    for (int i = 0 ; i < taskCommunicators.length ; i++) {
-      taskCommunicatorServiceWrappers[i].init(getConfig());
-      taskCommunicatorServiceWrappers[i].start();
-    }
-  }
-
-  @Override
-  public void serviceStop() {
-    for (int i = 0 ; i < taskCommunicators.length ; i++) {
-      taskCommunicatorServiceWrappers[i].stop();
-    }
-  }
-
-  @VisibleForTesting
-  TaskCommunicator createTaskCommunicator(NamedEntityDescriptor taskCommDescriptor,
-                                          int taskCommIndex) {
-    if (taskCommDescriptor.getEntityName().equals(TezConstants.getTezYarnServicePluginName())) {
-      return createDefaultTaskCommunicator(taskCommunicatorContexts[taskCommIndex]);
-    } else if (taskCommDescriptor.getEntityName()
-        .equals(TezConstants.getTezUberServicePluginName())) {
-      return createUberTaskCommunicator(taskCommunicatorContexts[taskCommIndex]);
-    } else {
-      return createCustomTaskCommunicator(taskCommunicatorContexts[taskCommIndex],
-          taskCommDescriptor);
-    }
-  }
-
-  @VisibleForTesting
-  TaskCommunicator createDefaultTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
-    LOG.info("Using Default Task Communicator");
-    return new TezTaskCommunicatorImpl(taskCommunicatorContext);
-  }
-
-  @VisibleForTesting
-  TaskCommunicator createUberTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
-    LOG.info("Using Default Local Task Communicator");
-    return new TezLocalTaskCommunicatorImpl(taskCommunicatorContext);
-  }
-
-  @VisibleForTesting
-  TaskCommunicator createCustomTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext,
-                                                NamedEntityDescriptor taskCommDescriptor) {
-    LOG.info("Using TaskCommunicator {}:{} " + taskCommDescriptor.getEntityName(),
-        taskCommDescriptor.getClassName());
-    Class<? extends TaskCommunicator> taskCommClazz =
-        (Class<? extends TaskCommunicator>) ReflectionUtils
-            .getClazz(taskCommDescriptor.getClassName());
-    try {
-      Constructor<? extends TaskCommunicator> ctor =
-          taskCommClazz.getConstructor(TaskCommunicatorContext.class);
-      return ctor.newInstance(taskCommunicatorContext);
-    } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
-      throw new TezUncheckedException(e);
-    }
-  }
-
-  public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request)
-      throws IOException, TezException {
-    ContainerId containerId = ConverterUtils.toContainerId(request
-        .getContainerIdentifier());
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Received heartbeat from container"
-          + ", request=" + request);
-    }
-
-    if (!registeredContainers.containsKey(containerId)) {
-      LOG.warn("Received task heartbeat from unknown container with id: " + containerId +
-          ", asking it to die");
-      return RESPONSE_SHOULD_DIE;
-    }
-
-    // A heartbeat can come in anytime. The AM may have made a decision to kill a running task/container
-    // meanwhile. If the decision is processed through the pipeline before the heartbeat is processed,
-    // the heartbeat will be dropped. Otherwise the heartbeat will be processed - and the system
-    // know how to handle this - via FailedInputEvents for example (relevant only if the heartbeat has events).
-    // So - avoiding synchronization.
-
-    pingContainerHeartbeatHandler(containerId);
-    TaskAttemptEventInfo eventInfo = new TaskAttemptEventInfo(0, null, 0);
-    TezTaskAttemptID taskAttemptID = request.getTaskAttemptId();
-    if (taskAttemptID != null) {
-      ContainerId containerIdFromMap = registeredAttempts.get(taskAttemptID);
-      if (containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
-        // This can happen when a task heartbeats. Meanwhile the container is unregistered.
-        // The information will eventually make it through to the plugin via a corresponding unregister.
-        // There's a race in that case between the unregister making it through, and this method returning.
-        // TODO TEZ-2003 (post) TEZ-2666. An exception back is likely a better approach than sending a shouldDie = true,
-        // so that the plugin can handle the scenario. Alternately augment the response with error codes.
-        // Error codes would be better than exceptions.
-        LOG.info("Attempt: " + taskAttemptID + " is not recognized for heartbeats");
-        return RESPONSE_SHOULD_DIE;
-      }
-
-      List<TezEvent> inEvents = request.getEvents();
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Ping from " + taskAttemptID.toString() +
-            " events: " + (inEvents != null ? inEvents.size() : -1));
-      }
-
-      long currTime = context.getClock().getTime();
-      List<TezEvent> otherEvents = new ArrayList<TezEvent>();
-      // route TASK_STATUS_UPDATE_EVENT directly to TaskAttempt and route other events
-      // (DATA_MOVEMENT_EVENT, TASK_ATTEMPT_COMPLETED_EVENT, TASK_ATTEMPT_FAILED_EVENT)
-      // to VertexImpl to ensure the events ordering
-      //  1. DataMovementEvent is logged as RecoveryEvent before TaskAttemptFinishedEvent
-      //  2. TaskStatusEvent is handled before TaskAttemptFinishedEvent
-      for (TezEvent tezEvent : ListUtils.emptyIfNull(inEvents)) {
-        // for now, set the event time on the AM when it is received.
-        // this avoids any time disparity between machines.
-        tezEvent.setEventReceivedTime(currTime);
-        final EventType eventType = tezEvent.getEventType();
-        if (eventType == EventType.TASK_STATUS_UPDATE_EVENT) {
-          TaskAttemptEvent taskAttemptEvent = new TaskAttemptEventStatusUpdate(taskAttemptID,
-              (TaskStatusUpdateEvent) tezEvent.getEvent());
-          context.getEventHandler().handle(taskAttemptEvent);
-        } else {
-          otherEvents.add(tezEvent);
-        }
-      }
-      if(!otherEvents.isEmpty()) {
-        TezVertexID vertexId = taskAttemptID.getTaskID().getVertexID();
-        context.getEventHandler().handle(
-            new VertexEventRouteEvent(vertexId, Collections.unmodifiableList(otherEvents)));
-      }
-      taskHeartbeatHandler.pinged(taskAttemptID);
-      eventInfo = context
-          .getCurrentDAG()
-          .getVertex(taskAttemptID.getTaskID().getVertexID())
-          .getTaskAttemptTezEvents(taskAttemptID, request.getStartIndex(), request.getPreRoutedStartIndex(),
-              request.getMaxEvents());
-    }
-    return new TaskHeartbeatResponse(false, eventInfo.getEvents(), eventInfo.getNextFromEventId(), eventInfo.getNextPreRoutedFromEventId());
-  }
-  public void taskAlive(TezTaskAttemptID taskAttemptId) {
-    taskHeartbeatHandler.pinged(taskAttemptId);
-  }
-
-  public void containerAlive(ContainerId containerId) {
-    pingContainerHeartbeatHandler(containerId);
-  }
-
-  public void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId) {
-    context.getEventHandler()
-        .handle(new TaskAttemptEventStartedRemotely(taskAttemptID, containerId, null));
-    pingContainerHeartbeatHandler(containerId);
-  }
-
-  public void taskKilled(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
-                         String diagnostics) {
-    // Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
-    // and messages from the scheduler will release the container.
-    // TODO TEZ-2003 (post) TEZ-2671 Maybe consider un-registering here itself, since the task is not active anymore,
-    // instead of waiting for the unregister to flow through the Container.
-    // Fix along the same lines as TEZ-2124 by introducing an explict context.
-    context.getEventHandler().handle(new TaskAttemptEventAttemptKilled(taskAttemptId,
-        diagnostics, TezUtilsInternal.fromTaskAttemptEndReason(
-        taskAttemptEndReason)));
-  }
-
-  public void taskFailed(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
-                         String diagnostics) {
-    // Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
-    // and messages from the scheduler will release the container.
-    // TODO TEZ-2003 (post) TEZ-2671 Maybe consider un-registering here itself, since the task is not active anymore,
-    // instead of waiting for the unregister to flow through the Container.
-    // Fix along the same lines as TEZ-2124 by introducing an explict context.
-    context.getEventHandler().handle(new TaskAttemptEventAttemptFailed(taskAttemptId,
-        TaskAttemptEventType.TA_FAILED, diagnostics, TezUtilsInternal.fromTaskAttemptEndReason(
-        taskAttemptEndReason)));
-  }
-
-  public void vertexStateUpdateNotificationReceived(VertexStateUpdate event, int taskCommIndex) throws
-      Exception {
-    taskCommunicators[taskCommIndex].onVertexStateUpdated(event);
-  }
-
-
-  /**
-   * Child checking whether it can commit.
-   * <p/>
-   * <br/>
-   * Repeatedly polls the ApplicationMaster whether it
-   * {@link Task#canCommit(TezTaskAttemptID)} This is * a legacy from the
-   * centralized commit protocol handling by the JobTracker.
-   */
-//  @Override
-  public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
-    LOG.info("Commit go/no-go request from " + taskAttemptId.toString());
-    // An attempt is asking if it can commit its output. This can be decided
-    // only by the task which is managing the multiple attempts. So redirect the
-    // request there.
-    taskHeartbeatHandler.progressing(taskAttemptId);
-    pingContainerHeartbeatHandler(taskAttemptId);
-
-    DAG job = context.getCurrentDAG();
-    Task task =
-        job.getVertex(taskAttemptId.getTaskID().getVertexID()).
-            getTask(taskAttemptId.getTaskID());
-    return task.canCommit(taskAttemptId);
-  }
-
-  // The TaskAttemptListener register / unregister methods in this class are not thread safe.
-  // The Tez framework should not invoke these methods from multiple threads.
-  @Override
-  public void dagComplete(DAG dag) {
-    // TODO TEZ-2335. Cleanup TaskHeartbeat handler structures.
-    // TODO TEZ-2345. Also cleanup attemptInfo map, so that any tasks which heartbeat are told to die.
-    // Container structures remain unchanged - since they could be re-used across restarts.
-    // This becomes more relevant when task kills without container kills are allowed.
-
-    // TODO TEZ-2336. Send a signal to containers indicating DAG completion.
-
-    // Inform all communicators of the dagCompletion.
-    for (int i = 0 ; i < taskCommunicators.length ; i++) {
-      ((TaskCommunicatorContextImpl)taskCommunicatorContexts[i]).dagCompleteStart(dag);
-      taskCommunicators[i].dagComplete(dag.getName());
-      ((TaskCommunicatorContextImpl)taskCommunicatorContexts[i]).dagCompleteEnd();
-    }
-
-  }
-
-  @Override
-  public void dagSubmitted() {
-    // Nothing to do right now. Indicates that a new DAG has been submitted and
-    // the context has updated information.
-  }
-
-  @Override
-  public void registerRunningContainer(ContainerId containerId, int taskCommId) {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("ContainerId: " + containerId + " registered with TaskAttemptListener");
-    }
-    ContainerInfo oldInfo = registeredContainers.put(containerId, NULL_CONTAINER_INFO);
-    if (oldInfo != null) {
-      throw new TezUncheckedException(
-          "Multiple registrations for containerId: " + containerId);
-    }
-    NodeId nodeId = context.getAllContainers().get(containerId).getContainer().getNodeId();
-    taskCommunicators[taskCommId].registerRunningContainer(containerId, nodeId.getHost(),
-        nodeId.getPort());
-  }
-
-  @Override
-  public void unregisterRunningContainer(ContainerId containerId, int taskCommId, ContainerEndReason endReason, String diagnostics) {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Unregistering Container from TaskAttemptListener: " + containerId);
-    }
-    ContainerInfo containerInfo = registeredContainers.remove(containerId);
-    if (containerInfo.taskAttemptId != null) {
-      registeredAttempts.remove(containerInfo.taskAttemptId);
-    }
-    taskCommunicators[taskCommId].registerContainerEnd(containerId, endReason, diagnostics);
-  }
-
-  @Override
-  public void registerTaskAttempt(AMContainerTask amContainerTask,
-                                  ContainerId containerId, int taskCommId) {
-    ContainerInfo containerInfo = registeredContainers.get(containerId);
-    if (containerInfo == null) {
-      throw new TezUncheckedException("Registering task attempt: "
-          + amContainerTask.getTask().getTaskAttemptID() + " to unknown container: " + containerId);
-    }
-    if (containerInfo.taskAttemptId != null) {
-      throw new TezUncheckedException("Registering task attempt: "
-          + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
-          + " with existing assignment to: " +
-          containerInfo.taskAttemptId);
-    }
-
-    // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
-    registeredContainers.put(containerId, new ContainerInfo(amContainerTask.getTask().getTaskAttemptID()));
-
-    ContainerId containerIdFromMap = registeredAttempts.put(
-        amContainerTask.getTask().getTaskAttemptID(), containerId);
-    if (containerIdFromMap != null) {
-      throw new TezUncheckedException("Registering task attempt: "
-          + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
-          + " when already assigned to: " + containerIdFromMap);
-    }
-    taskCommunicators[taskCommId].registerRunningTaskAttempt(containerId, amContainerTask.getTask(),
-        amContainerTask.getAdditionalResources(), amContainerTask.getCredentials(),
-        amContainerTask.haveCredentialsChanged(), amContainerTask.getPriority());
-  }
-
-  @Override
-  public void unregisterTaskAttempt(TezTaskAttemptID attemptId, int taskCommId, TaskAttemptEndReason endReason, String diagnostics) {
-    ContainerId containerId = registeredAttempts.remove(attemptId);
-    if (containerId == null) {
-      LOG.warn("Unregister task attempt: " + attemptId + " from unknown container");
-      return;
-    }
-    ContainerInfo containerInfo = registeredContainers.get(containerId);
-    if (containerInfo == null) {
-      LOG.warn("Unregister task attempt: " + attemptId +
-          " from non-registered container: " + containerId);
-      return;
-    }
-    // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
-    registeredContainers.put(containerId, NULL_CONTAINER_INFO);
-    taskCommunicators[taskCommId].unregisterRunningTaskAttempt(attemptId, endReason, diagnostics);
-  }
-
-  @Override
-  public TaskCommunicator getTaskCommunicator(int taskCommIndex) {
-    return taskCommunicators[taskCommIndex];
-  }
-
-  private void pingContainerHeartbeatHandler(ContainerId containerId) {
-    containerHeartbeatHandler.pinged(containerId);
-  }
-
-  private void pingContainerHeartbeatHandler(TezTaskAttemptID taskAttemptId) {
-    ContainerId containerId = registeredAttempts.get(taskAttemptId);
-    if (containerId != null) {
-      containerHeartbeatHandler.pinged(containerId);
-    } else {
-      LOG.warn("Handling communication from attempt: " + taskAttemptId
-          + ", ContainerId not known for this attempt");
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
index 9d57ac3..071b008 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
@@ -47,7 +47,7 @@ public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, Ver
   // TODO TEZ-2003 (post) TEZ-2669 Propagate errors baack to the AM with proper error reporting
 
   private final AppContext context;
-  private final TaskAttemptListenerImpTezDag taskAttemptListener;
+  private final TaskCommunicatorManager taskCommunicatorManager;
   private final int taskCommunicatorIndex;
   private final ReentrantReadWriteLock.ReadLock dagChangedReadLock;
   private final ReentrantReadWriteLock.WriteLock dagChangedWriteLock;
@@ -56,11 +56,11 @@ public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, Ver
   private DAG dag;
 
   public TaskCommunicatorContextImpl(AppContext appContext,
-                                     TaskAttemptListenerImpTezDag taskAttemptListener,
+                                     TaskCommunicatorManager taskCommunicatorManager,
                                      UserPayload userPayload,
                                      int taskCommunicatorIndex) {
     this.context = appContext;
-    this.taskAttemptListener = taskAttemptListener;
+    this.taskCommunicatorManager = taskCommunicatorManager;
     this.userPayload = userPayload;
     this.taskCommunicatorIndex = taskCommunicatorIndex;
 
@@ -86,13 +86,13 @@ public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, Ver
 
   @Override
   public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
-    return taskAttemptListener.canCommit(taskAttemptId);
+    return taskCommunicatorManager.canCommit(taskAttemptId);
   }
 
   @Override
   public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request) throws IOException,
       TezException {
-    return taskAttemptListener.heartbeat(request);
+    return taskCommunicatorManager.heartbeat(request);
   }
 
   @Override
@@ -108,31 +108,31 @@ public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, Ver
 
   @Override
   public void taskAlive(TezTaskAttemptID taskAttemptId) {
-    taskAttemptListener.taskAlive(taskAttemptId);
+    taskCommunicatorManager.taskAlive(taskAttemptId);
   }
 
   @Override
   public void containerAlive(ContainerId containerId) {
     if (isKnownContainer(containerId)) {
-      taskAttemptListener.containerAlive(containerId);
+      taskCommunicatorManager.containerAlive(containerId);
     }
   }
 
   @Override
   public void taskStartedRemotely(TezTaskAttemptID taskAttemptId, ContainerId containerId) {
-    taskAttemptListener.taskStartedRemotely(taskAttemptId, containerId);
+    taskCommunicatorManager.taskStartedRemotely(taskAttemptId, containerId);
   }
 
   @Override
   public void taskKilled(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
                          @Nullable String diagnostics) {
-    taskAttemptListener.taskKilled(taskAttemptId, taskAttemptEndReason, diagnostics);
+    taskCommunicatorManager.taskKilled(taskAttemptId, taskAttemptEndReason, diagnostics);
   }
 
   @Override
   public void taskFailed(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
                          @Nullable String diagnostics) {
-    taskAttemptListener.taskFailed(taskAttemptId, taskAttemptEndReason, diagnostics);
+    taskCommunicatorManager.taskFailed(taskAttemptId, taskAttemptEndReason, diagnostics);
 
   }
 
@@ -196,7 +196,7 @@ public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, Ver
   @Override
   public void onStateUpdated(VertexStateUpdate event) {
     try {
-      taskAttemptListener.vertexStateUpdateNotificationReceived(event, taskCommunicatorIndex);
+      taskCommunicatorManager.vertexStateUpdateNotificationReceived(event, taskCommunicatorIndex);
     } catch (Exception e) {
       throw new TezUncheckedException(e);
     }

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java
new file mode 100644
index 0000000..42df259
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java
@@ -0,0 +1,449 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.tez.dag.app;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.commons.collections4.ListUtils;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
+import org.apache.tez.serviceplugins.api.ContainerEndReason;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEvent;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate;
+import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent;
+import org.apache.tez.runtime.api.impl.EventType;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.dag.api.TaskCommunicator;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.dag.api.TaskHeartbeatResponse;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.dag.api.TaskHeartbeatRequest;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.Task;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptKilled;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
+import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
+import org.apache.tez.dag.app.rm.container.AMContainerTask;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.api.impl.TezEvent;
+
+
+@SuppressWarnings("unchecked")
+@InterfaceAudience.Private
+public class TaskCommunicatorManager extends AbstractService implements
+    TaskCommunicatorManagerInterface {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(TaskCommunicatorManager.class);
+
+  private final AppContext context;
+  private final TaskCommunicator[] taskCommunicators;
+  private final TaskCommunicatorContext[] taskCommunicatorContexts;
+  protected final ServicePluginLifecycleAbstractService []taskCommunicatorServiceWrappers;
+
+  protected final TaskHeartbeatHandler taskHeartbeatHandler;
+  protected final ContainerHeartbeatHandler containerHeartbeatHandler;
+
+  private final TaskHeartbeatResponse RESPONSE_SHOULD_DIE = new TaskHeartbeatResponse(true, null, 0, 0);
+
+  private final ConcurrentMap<TezTaskAttemptID, ContainerId> registeredAttempts =
+      new ConcurrentHashMap<TezTaskAttemptID, ContainerId>();
+  private final ConcurrentMap<ContainerId, ContainerInfo> registeredContainers =
+      new ConcurrentHashMap<ContainerId, ContainerInfo>();
+
+  // Defined primarily to work around ConcurrentMaps not accepting null values
+  private static final class ContainerInfo {
+    TezTaskAttemptID taskAttemptId;
+    ContainerInfo(TezTaskAttemptID taskAttemptId) {
+      this.taskAttemptId = taskAttemptId;
+    }
+  }
+
+  private static final ContainerInfo NULL_CONTAINER_INFO = new ContainerInfo(null);
+
+
+  public TaskCommunicatorManager(AppContext context,
+                                 TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
+                                 List<NamedEntityDescriptor> taskCommunicatorDescriptors) {
+    super(TaskCommunicatorManager.class.getName());
+    this.context = context;
+    this.taskHeartbeatHandler = thh;
+    this.containerHeartbeatHandler = chh;
+    Preconditions.checkArgument(
+        taskCommunicatorDescriptors != null && !taskCommunicatorDescriptors.isEmpty(),
+        "TaskCommunicators must be specified");
+    this.taskCommunicators = new TaskCommunicator[taskCommunicatorDescriptors.size()];
+    this.taskCommunicatorContexts = new TaskCommunicatorContext[taskCommunicatorDescriptors.size()];
+    this.taskCommunicatorServiceWrappers = new ServicePluginLifecycleAbstractService[taskCommunicatorDescriptors.size()];
+    for (int i = 0 ; i < taskCommunicatorDescriptors.size() ; i++) {
+      UserPayload userPayload = taskCommunicatorDescriptors.get(i).getUserPayload();
+      taskCommunicatorContexts[i] = new TaskCommunicatorContextImpl(context, this, userPayload, i);
+      taskCommunicators[i] = createTaskCommunicator(taskCommunicatorDescriptors.get(i), i);
+      taskCommunicatorServiceWrappers[i] = new ServicePluginLifecycleAbstractService(taskCommunicators[i]);
+    }
+    // TODO TEZ-2118 Start using taskCommunicator indices properly
+  }
+
+  @Override
+  public void serviceStart() {
+    // TODO Why is init tied to serviceStart
+    for (int i = 0 ; i < taskCommunicators.length ; i++) {
+      taskCommunicatorServiceWrappers[i].init(getConfig());
+      taskCommunicatorServiceWrappers[i].start();
+    }
+  }
+
+  @Override
+  public void serviceStop() {
+    for (int i = 0 ; i < taskCommunicators.length ; i++) {
+      taskCommunicatorServiceWrappers[i].stop();
+    }
+  }
+
+  @VisibleForTesting
+  TaskCommunicator createTaskCommunicator(NamedEntityDescriptor taskCommDescriptor,
+                                          int taskCommIndex) {
+    if (taskCommDescriptor.getEntityName().equals(TezConstants.getTezYarnServicePluginName())) {
+      return createDefaultTaskCommunicator(taskCommunicatorContexts[taskCommIndex]);
+    } else if (taskCommDescriptor.getEntityName()
+        .equals(TezConstants.getTezUberServicePluginName())) {
+      return createUberTaskCommunicator(taskCommunicatorContexts[taskCommIndex]);
+    } else {
+      return createCustomTaskCommunicator(taskCommunicatorContexts[taskCommIndex],
+          taskCommDescriptor);
+    }
+  }
+
+  @VisibleForTesting
+  TaskCommunicator createDefaultTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
+    LOG.info("Using Default Task Communicator");
+    return new TezTaskCommunicatorImpl(taskCommunicatorContext);
+  }
+
+  @VisibleForTesting
+  TaskCommunicator createUberTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
+    LOG.info("Using Default Local Task Communicator");
+    return new TezLocalTaskCommunicatorImpl(taskCommunicatorContext);
+  }
+
+  @VisibleForTesting
+  TaskCommunicator createCustomTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext,
+                                                NamedEntityDescriptor taskCommDescriptor) {
+    LOG.info("Using TaskCommunicator {}:{} " + taskCommDescriptor.getEntityName(),
+        taskCommDescriptor.getClassName());
+    Class<? extends TaskCommunicator> taskCommClazz =
+        (Class<? extends TaskCommunicator>) ReflectionUtils
+            .getClazz(taskCommDescriptor.getClassName());
+    try {
+      Constructor<? extends TaskCommunicator> ctor =
+          taskCommClazz.getConstructor(TaskCommunicatorContext.class);
+      return ctor.newInstance(taskCommunicatorContext);
+    } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
+      throw new TezUncheckedException(e);
+    }
+  }
+
+  public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request)
+      throws IOException, TezException {
+    ContainerId containerId = ConverterUtils.toContainerId(request
+        .getContainerIdentifier());
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Received heartbeat from container"
+          + ", request=" + request);
+    }
+
+    if (!registeredContainers.containsKey(containerId)) {
+      LOG.warn("Received task heartbeat from unknown container with id: " + containerId +
+          ", asking it to die");
+      return RESPONSE_SHOULD_DIE;
+    }
+
+    // A heartbeat can come in anytime. The AM may have made a decision to kill a running task/container
+    // meanwhile. If the decision is processed through the pipeline before the heartbeat is processed,
+    // the heartbeat will be dropped. Otherwise the heartbeat will be processed - and the system
+    // know how to handle this - via FailedInputEvents for example (relevant only if the heartbeat has events).
+    // So - avoiding synchronization.
+
+    pingContainerHeartbeatHandler(containerId);
+    TaskAttemptEventInfo eventInfo = new TaskAttemptEventInfo(0, null, 0);
+    TezTaskAttemptID taskAttemptID = request.getTaskAttemptId();
+    if (taskAttemptID != null) {
+      ContainerId containerIdFromMap = registeredAttempts.get(taskAttemptID);
+      if (containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
+        // This can happen when a task heartbeats. Meanwhile the container is unregistered.
+        // The information will eventually make it through to the plugin via a corresponding unregister.
+        // There's a race in that case between the unregister making it through, and this method returning.
+        // TODO TEZ-2003 (post) TEZ-2666. An exception back is likely a better approach than sending a shouldDie = true,
+        // so that the plugin can handle the scenario. Alternately augment the response with error codes.
+        // Error codes would be better than exceptions.
+        LOG.info("Attempt: " + taskAttemptID + " is not recognized for heartbeats");
+        return RESPONSE_SHOULD_DIE;
+      }
+
+      List<TezEvent> inEvents = request.getEvents();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Ping from " + taskAttemptID.toString() +
+            " events: " + (inEvents != null ? inEvents.size() : -1));
+      }
+
+      long currTime = context.getClock().getTime();
+      List<TezEvent> otherEvents = new ArrayList<TezEvent>();
+      // route TASK_STATUS_UPDATE_EVENT directly to TaskAttempt and route other events
+      // (DATA_MOVEMENT_EVENT, TASK_ATTEMPT_COMPLETED_EVENT, TASK_ATTEMPT_FAILED_EVENT)
+      // to VertexImpl to ensure the events ordering
+      //  1. DataMovementEvent is logged as RecoveryEvent before TaskAttemptFinishedEvent
+      //  2. TaskStatusEvent is handled before TaskAttemptFinishedEvent
+      for (TezEvent tezEvent : ListUtils.emptyIfNull(inEvents)) {
+        // for now, set the event time on the AM when it is received.
+        // this avoids any time disparity between machines.
+        tezEvent.setEventReceivedTime(currTime);
+        final EventType eventType = tezEvent.getEventType();
+        if (eventType == EventType.TASK_STATUS_UPDATE_EVENT) {
+          TaskAttemptEvent taskAttemptEvent = new TaskAttemptEventStatusUpdate(taskAttemptID,
+              (TaskStatusUpdateEvent) tezEvent.getEvent());
+          context.getEventHandler().handle(taskAttemptEvent);
+        } else {
+          otherEvents.add(tezEvent);
+        }
+      }
+      if(!otherEvents.isEmpty()) {
+        TezVertexID vertexId = taskAttemptID.getTaskID().getVertexID();
+        context.getEventHandler().handle(
+            new VertexEventRouteEvent(vertexId, Collections.unmodifiableList(otherEvents)));
+      }
+      taskHeartbeatHandler.pinged(taskAttemptID);
+      eventInfo = context
+          .getCurrentDAG()
+          .getVertex(taskAttemptID.getTaskID().getVertexID())
+          .getTaskAttemptTezEvents(taskAttemptID, request.getStartIndex(), request.getPreRoutedStartIndex(),
+              request.getMaxEvents());
+    }
+    return new TaskHeartbeatResponse(false, eventInfo.getEvents(), eventInfo.getNextFromEventId(), eventInfo.getNextPreRoutedFromEventId());
+  }
+  public void taskAlive(TezTaskAttemptID taskAttemptId) {
+    taskHeartbeatHandler.pinged(taskAttemptId);
+  }
+
+  public void containerAlive(ContainerId containerId) {
+    pingContainerHeartbeatHandler(containerId);
+  }
+
+  public void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId) {
+    context.getEventHandler()
+        .handle(new TaskAttemptEventStartedRemotely(taskAttemptID, containerId, null));
+    pingContainerHeartbeatHandler(containerId);
+  }
+
+  public void taskKilled(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
+                         String diagnostics) {
+    // Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
+    // and messages from the scheduler will release the container.
+    // TODO TEZ-2003 (post) TEZ-2671 Maybe consider un-registering here itself, since the task is not active anymore,
+    // instead of waiting for the unregister to flow through the Container.
+    // Fix along the same lines as TEZ-2124 by introducing an explict context.
+    context.getEventHandler().handle(new TaskAttemptEventAttemptKilled(taskAttemptId,
+        diagnostics, TezUtilsInternal.fromTaskAttemptEndReason(
+        taskAttemptEndReason)));
+  }
+
+  public void taskFailed(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
+                         String diagnostics) {
+    // Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
+    // and messages from the scheduler will release the container.
+    // TODO TEZ-2003 (post) TEZ-2671 Maybe consider un-registering here itself, since the task is not active anymore,
+    // instead of waiting for the unregister to flow through the Container.
+    // Fix along the same lines as TEZ-2124 by introducing an explict context.
+    context.getEventHandler().handle(new TaskAttemptEventAttemptFailed(taskAttemptId,
+        TaskAttemptEventType.TA_FAILED, diagnostics, TezUtilsInternal.fromTaskAttemptEndReason(
+        taskAttemptEndReason)));
+  }
+
+  public void vertexStateUpdateNotificationReceived(VertexStateUpdate event, int taskCommIndex) throws
+      Exception {
+    taskCommunicators[taskCommIndex].onVertexStateUpdated(event);
+  }
+
+
+  /**
+   * Child checking whether it can commit.
+   * <p/>
+   * <br/>
+   * Repeatedly polls the ApplicationMaster whether it
+   * {@link Task#canCommit(TezTaskAttemptID)} This is * a legacy from the
+   * centralized commit protocol handling by the JobTracker.
+   */
+//  @Override
+  public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
+    LOG.info("Commit go/no-go request from " + taskAttemptId.toString());
+    // An attempt is asking if it can commit its output. This can be decided
+    // only by the task which is managing the multiple attempts. So redirect the
+    // request there.
+    taskHeartbeatHandler.progressing(taskAttemptId);
+    pingContainerHeartbeatHandler(taskAttemptId);
+
+    DAG job = context.getCurrentDAG();
+    Task task =
+        job.getVertex(taskAttemptId.getTaskID().getVertexID()).
+            getTask(taskAttemptId.getTaskID());
+    return task.canCommit(taskAttemptId);
+  }
+
+  // The TaskAttemptListener register / unregister methods in this class are not thread safe.
+  // The Tez framework should not invoke these methods from multiple threads.
+  @Override
+  public void dagComplete(DAG dag) {
+    // TODO TEZ-2335. Cleanup TaskHeartbeat handler structures.
+    // TODO TEZ-2345. Also cleanup attemptInfo map, so that any tasks which heartbeat are told to die.
+    // Container structures remain unchanged - since they could be re-used across restarts.
+    // This becomes more relevant when task kills without container kills are allowed.
+
+    // TODO TEZ-2336. Send a signal to containers indicating DAG completion.
+
+    // Inform all communicators of the dagCompletion.
+    for (int i = 0 ; i < taskCommunicators.length ; i++) {
+      ((TaskCommunicatorContextImpl)taskCommunicatorContexts[i]).dagCompleteStart(dag);
+      taskCommunicators[i].dagComplete(dag.getName());
+      ((TaskCommunicatorContextImpl)taskCommunicatorContexts[i]).dagCompleteEnd();
+    }
+
+  }
+
+  @Override
+  public void dagSubmitted() {
+    // Nothing to do right now. Indicates that a new DAG has been submitted and
+    // the context has updated information.
+  }
+
+  @Override
+  public void registerRunningContainer(ContainerId containerId, int taskCommId) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("ContainerId: " + containerId + " registered with TaskAttemptListener");
+    }
+    ContainerInfo oldInfo = registeredContainers.put(containerId, NULL_CONTAINER_INFO);
+    if (oldInfo != null) {
+      throw new TezUncheckedException(
+          "Multiple registrations for containerId: " + containerId);
+    }
+    NodeId nodeId = context.getAllContainers().get(containerId).getContainer().getNodeId();
+    taskCommunicators[taskCommId].registerRunningContainer(containerId, nodeId.getHost(),
+        nodeId.getPort());
+  }
+
+  @Override
+  public void unregisterRunningContainer(ContainerId containerId, int taskCommId, ContainerEndReason endReason, String diagnostics) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Unregistering Container from TaskAttemptListener: " + containerId);
+    }
+    ContainerInfo containerInfo = registeredContainers.remove(containerId);
+    if (containerInfo.taskAttemptId != null) {
+      registeredAttempts.remove(containerInfo.taskAttemptId);
+    }
+    taskCommunicators[taskCommId].registerContainerEnd(containerId, endReason, diagnostics);
+  }
+
+  @Override
+  public void registerTaskAttempt(AMContainerTask amContainerTask,
+                                  ContainerId containerId, int taskCommId) {
+    ContainerInfo containerInfo = registeredContainers.get(containerId);
+    if (containerInfo == null) {
+      throw new TezUncheckedException("Registering task attempt: "
+          + amContainerTask.getTask().getTaskAttemptID() + " to unknown container: " + containerId);
+    }
+    if (containerInfo.taskAttemptId != null) {
+      throw new TezUncheckedException("Registering task attempt: "
+          + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+          + " with existing assignment to: " +
+          containerInfo.taskAttemptId);
+    }
+
+    // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
+    registeredContainers.put(containerId, new ContainerInfo(amContainerTask.getTask().getTaskAttemptID()));
+
+    ContainerId containerIdFromMap = registeredAttempts.put(
+        amContainerTask.getTask().getTaskAttemptID(), containerId);
+    if (containerIdFromMap != null) {
+      throw new TezUncheckedException("Registering task attempt: "
+          + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+          + " when already assigned to: " + containerIdFromMap);
+    }
+    taskCommunicators[taskCommId].registerRunningTaskAttempt(containerId, amContainerTask.getTask(),
+        amContainerTask.getAdditionalResources(), amContainerTask.getCredentials(),
+        amContainerTask.haveCredentialsChanged(), amContainerTask.getPriority());
+  }
+
+  @Override
+  public void unregisterTaskAttempt(TezTaskAttemptID attemptId, int taskCommId, TaskAttemptEndReason endReason, String diagnostics) {
+    ContainerId containerId = registeredAttempts.remove(attemptId);
+    if (containerId == null) {
+      LOG.warn("Unregister task attempt: " + attemptId + " from unknown container");
+      return;
+    }
+    ContainerInfo containerInfo = registeredContainers.get(containerId);
+    if (containerInfo == null) {
+      LOG.warn("Unregister task attempt: " + attemptId +
+          " from non-registered container: " + containerId);
+      return;
+    }
+    // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
+    registeredContainers.put(containerId, NULL_CONTAINER_INFO);
+    taskCommunicators[taskCommId].unregisterRunningTaskAttempt(attemptId, endReason, diagnostics);
+  }
+
+  @Override
+  public TaskCommunicator getTaskCommunicator(int taskCommIndex) {
+    return taskCommunicators[taskCommIndex];
+  }
+
+  private void pingContainerHeartbeatHandler(ContainerId containerId) {
+    containerHeartbeatHandler.pinged(containerId);
+  }
+
+  private void pingContainerHeartbeatHandler(TezTaskAttemptID taskAttemptId) {
+    ContainerId containerId = registeredAttempts.get(taskAttemptId);
+    if (containerId != null) {
+      containerHeartbeatHandler.pinged(containerId);
+    } else {
+      LOG.warn("Handling communication from attempt: " + taskAttemptId
+          + ", ContainerId not known for this attempt");
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java
new file mode 100644
index 0000000..8d060a2
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java
@@ -0,0 +1,46 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.app;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.serviceplugins.api.ContainerEndReason;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.api.TaskCommunicator;
+import org.apache.tez.dag.app.rm.container.AMContainerTask;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+/**
+ * This class listens for changes to the state of a Task.
+ */
+public interface TaskCommunicatorManagerInterface {
+
+  void registerRunningContainer(ContainerId containerId, int taskCommId);
+
+  void registerTaskAttempt(AMContainerTask amContainerTask, ContainerId containerId, int taskCommId);
+  
+  void unregisterRunningContainer(ContainerId containerId, int taskCommId, ContainerEndReason endReason, String diagnostics);
+  
+  void unregisterTaskAttempt(TezTaskAttemptID attemptID, int taskCommId, TaskAttemptEndReason endReason, String diagnostics);
+
+  void dagComplete(DAG dag);
+
+  void dagSubmitted();
+
+  TaskCommunicator getTaskCommunicator(int taskCommIndex);
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 6b474ff..756ed28 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -82,7 +82,7 @@ import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
 import org.apache.tez.dag.api.records.DAGProtos.PlanVertexGroupInfo;
 import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.DAGReport;
@@ -160,7 +160,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   private final Lock readLock;
   private final Lock writeLock;
   private final String dagName;
-  private final TaskAttemptListener taskAttemptListener;
+  private final TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
   private final TaskHeartbeatHandler taskHeartbeatHandler;
   private final Object tasksSyncHandle = new Object();
 
@@ -489,7 +489,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
       Configuration amConf,
       DAGPlan jobPlan,
       EventHandler eventHandler,
-      TaskAttemptListener taskAttemptListener,
+      TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
       Credentials dagCredentials,
       Clock clock,
       String appUserName,
@@ -511,7 +511,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
     this.clock = clock;
     this.appContext = appContext;
 
-    this.taskAttemptListener = taskAttemptListener;
+    this.taskCommunicatorManagerInterface = taskCommunicatorManagerInterface;
     this.taskHeartbeatHandler = thh;
     this.eventHandler = eventHandler;
     ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
@@ -1538,7 +1538,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
 
     VertexImpl v = new VertexImpl(
         vertexId, vertexPlan, vertexName, dag.dagConf,
-        dag.eventHandler, dag.taskAttemptListener,
+        dag.eventHandler, dag.taskCommunicatorManagerInterface,
         dag.clock, dag.taskHeartbeatHandler,
         !dag.commitAllOutputsOnSuccess, dag.appContext, vertexLocationHint,
         dag.vertexGroups, dag.taskSpecificLaunchCmdOption, dag.entityUpdateTracker);

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 6957b1d..3f2e3a4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -61,7 +61,7 @@ import org.apache.tez.dag.api.oldrecords.TaskAttemptReport;
 import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.Task;
 import org.apache.tez.dag.app.dag.TaskAttempt;
@@ -403,17 +403,17 @@ public class TaskAttemptImpl implements TaskAttempt,
 
   @SuppressWarnings("rawtypes")
   public TaskAttemptImpl(TezTaskID taskId, int attemptNumber, EventHandler eventHandler,
-      TaskAttemptListener taskAttemptListener, Configuration conf, Clock clock,
+      TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock,
       TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
       boolean isRescheduled,
       Resource resource, ContainerContext containerContext, boolean leafVertex,
       Task task) {
-    this(taskId, attemptNumber, eventHandler, taskAttemptListener, conf, clock,
+    this(taskId, attemptNumber, eventHandler, taskCommunicatorManagerInterface, conf, clock,
         taskHeartbeatHandler, appContext, isRescheduled, resource, containerContext, leafVertex,
         task, null);
   }
   public TaskAttemptImpl(TezTaskID taskId, int attemptNumber, EventHandler eventHandler,
-      TaskAttemptListener taskAttemptListener, Configuration conf, Clock clock,
+      TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock,
       TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
       boolean isRescheduled,
       Resource resource, ContainerContext containerContext, boolean leafVertex,

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 1b55295..ea14483 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -56,7 +56,7 @@ import org.apache.tez.dag.api.oldrecords.TaskReport;
 import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.StateChangeNotifier;
 import org.apache.tez.dag.app.dag.Task;
@@ -115,7 +115,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     .getProperty("line.separator");
 
   protected final Configuration conf;
-  protected final TaskAttemptListener taskAttemptListener;
+  protected final TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
   protected final TaskHeartbeatHandler taskHeartbeatHandler;
   protected final EventHandler eventHandler;
   private final TezTaskID taskId;
@@ -341,7 +341,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
 
   public TaskImpl(TezVertexID vertexId, int taskIndex,
       EventHandler eventHandler, Configuration conf,
-      TaskAttemptListener taskAttemptListener,
+      TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
       Clock clock, TaskHeartbeatHandler thh, AppContext appContext,
       boolean leafVertex, Resource resource,
       ContainerContext containerContext,
@@ -357,7 +357,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
     maxFailedAttempts = this.conf.getInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS,
                               TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS_DEFAULT);
     taskId = TezTaskID.getInstance(vertexId, taskIndex);
-    this.taskAttemptListener = taskAttemptListener;
+    this.taskCommunicatorManagerInterface = taskCommunicatorManagerInterface;
     this.taskHeartbeatHandler = thh;
     this.eventHandler = eventHandler;
     this.appContext = appContext;
@@ -817,7 +817,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
   
   TaskAttemptImpl createAttempt(int attemptNumber, TezTaskAttemptID schedulingCausalTA) {
     return new TaskAttemptImpl(getTaskId(), attemptNumber, eventHandler,
-        taskAttemptListener, conf, clock, taskHeartbeatHandler, appContext,
+        taskCommunicatorManagerInterface, conf, clock, taskHeartbeatHandler, appContext,
         (failedAttempts > 0), taskResource, containerContext, leafVertex, this, schedulingCausalTA);
   }
 

http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 3cc439f..a1dcf6c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -94,7 +94,7 @@ import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.ContainerContext;
 import org.apache.tez.dag.app.TaskAttemptEventInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
 import org.apache.tez.dag.app.TaskHeartbeatHandler;
 import org.apache.tez.dag.app.dag.DAG;
 import org.apache.tez.dag.app.dag.RootInputInitializerManager;
@@ -212,7 +212,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl
 
   private final Lock readLock;
   private final Lock writeLock;
-  private final TaskAttemptListener taskAttemptListener;
+  private final TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
   private final TaskHeartbeatHandler taskHeartbeatHandler;
   private final Object tasksSyncHandle = new Object();
 
@@ -890,7 +890,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl
 
   public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan,
       String vertexName, Configuration dagConf, EventHandler eventHandler,
-      TaskAttemptListener taskAttemptListener, Clock clock,
+      TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Clock clock,
       TaskHeartbeatHandler thh, boolean commitVertexOutputs,
       AppContext appContext, VertexLocationHint vertexLocationHint,
       Map<String, VertexGroupInfo> dagVertexGroups, TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption,
@@ -911,7 +911,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl
     this.appContext = appContext;
     this.commitVertexOutputs = commitVertexOutputs;
 
-    this.taskAttemptListener = taskAttemptListener;
+    this.taskCommunicatorManagerInterface = taskCommunicatorManagerInterface;
     this.taskHeartbeatHandler = thh;
     this.eventHandler = eventHandler;
     ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
@@ -2331,7 +2331,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl
     return new TaskImpl(this.getVertexId(), taskIndex,
         this.eventHandler,
         vertexConf,
-        this.taskAttemptListener,
+        this.taskCommunicatorManagerInterface,
         this.clock,
         this.taskHeartbeatHandler,
         this.appContext,