You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2015/08/26 01:48:48 UTC
[1/7] tez git commit: TEZ-2708. Rename classes and variables post
TEZ-2003 changes. (sseth)
Repository: tez
Updated Branches:
refs/heads/master dc0ee0115 -> 8b278ea84
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
new file mode 100644
index 0000000..98b7baa
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
@@ -0,0 +1,708 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.ContainerSignatureMatcher;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TaskLocationHint;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.client.DAGClientServer;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.ContainerContext;
+import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
+import org.apache.tez.dag.app.dag.TaskAttempt;
+import org.apache.tez.dag.app.dag.impl.TaskAttemptImpl;
+import org.apache.tez.dag.app.dag.impl.TaskImpl;
+import org.apache.tez.dag.app.dag.impl.VertexImpl;
+import org.apache.tez.dag.app.rm.container.AMContainer;
+import org.apache.tez.dag.app.rm.container.AMContainerEventAssignTA;
+import org.apache.tez.dag.app.rm.container.AMContainerEventCompleted;
+import org.apache.tez.dag.app.rm.container.AMContainerEventType;
+import org.apache.tez.dag.app.rm.container.AMContainerMap;
+import org.apache.tez.dag.app.rm.container.AMContainerState;
+import org.apache.tez.dag.app.web.WebUIService;
+import org.apache.tez.dag.records.TaskAttemptTerminationCause;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.serviceplugins.api.TaskScheduler;
+import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+@SuppressWarnings("rawtypes")
+public class TestTaskSchedulerManager {
+
+ class TestEventHandler implements EventHandler{
+ List<Event> events = Lists.newLinkedList();
+ @Override
+ public void handle(Event event) {
+ events.add(event);
+ }
+ }
+
+ class MockTaskSchedulerManager extends TaskSchedulerManager {
+
+ final AtomicBoolean notify = new AtomicBoolean(false);
+
+ public MockTaskSchedulerManager(AppContext appContext,
+ DAGClientServer clientService, EventHandler eventHandler,
+ ContainerSignatureMatcher containerSignatureMatcher,
+ WebUIService webUI) {
+ super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI,
+ Lists.newArrayList(new NamedEntityDescriptor("FakeDescriptor", null)), false);
+ }
+
+ @Override
+ protected void instantiateSchedulers(String host, int port, String trackingUrl,
+ AppContext appContext) {
+ taskSchedulers[0] = mockTaskScheduler;
+ taskSchedulerServiceWrappers[0] = new ServicePluginLifecycleAbstractService<>(taskSchedulers[0]);
+ }
+
+ @Override
+ protected void notifyForTest() {
+ synchronized (notify) {
+ notify.set(true);
+ notify.notifyAll();
+ }
+ }
+
+ }
+
+ AppContext mockAppContext;
+ DAGClientServer mockClientService;
+ TestEventHandler mockEventHandler;
+ ContainerSignatureMatcher mockSigMatcher;
+ MockTaskSchedulerManager schedulerHandler;
+ TaskScheduler mockTaskScheduler;
+ AMContainerMap mockAMContainerMap;
+ WebUIService mockWebUIService;
+
+ @Before
+ public void setup() {
+ mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
+ doReturn(new Configuration(false)).when(mockAppContext).getAMConf();
+ mockClientService = mock(DAGClientServer.class);
+ mockEventHandler = new TestEventHandler();
+ mockSigMatcher = mock(ContainerSignatureMatcher.class);
+ mockTaskScheduler = mock(TaskScheduler.class);
+ mockAMContainerMap = mock(AMContainerMap.class);
+ mockWebUIService = mock(WebUIService.class);
+ when(mockAppContext.getAllContainers()).thenReturn(mockAMContainerMap);
+ when(mockClientService.getBindAddress()).thenReturn(new InetSocketAddress(10000));
+ schedulerHandler = new MockTaskSchedulerManager(
+ mockAppContext, mockClientService, mockEventHandler, mockSigMatcher, mockWebUIService);
+ }
+
+ @Test(timeout = 5000)
+ public void testSimpleAllocate() throws Exception {
+ Configuration conf = new Configuration(false);
+ schedulerHandler.init(conf);
+ schedulerHandler.start();
+
+ TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
+ TezTaskAttemptID mockAttemptId = mock(TezTaskAttemptID.class);
+ when(mockAttemptId.getId()).thenReturn(0);
+ when(mockTaskAttempt.getID()).thenReturn(mockAttemptId);
+ Resource resource = Resource.newInstance(1024, 1);
+ ContainerContext containerContext =
+ new ContainerContext(new HashMap<String, LocalResource>(), new Credentials(),
+ new HashMap<String, String>(), "");
+ int priority = 10;
+ TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(), null);
+
+ ContainerId mockCId = mock(ContainerId.class);
+ Container container = mock(Container.class);
+ when(container.getId()).thenReturn(mockCId);
+
+ AMContainer mockAMContainer = mock(AMContainer.class);
+ when(mockAMContainer.getContainerId()).thenReturn(mockCId);
+ when(mockAMContainer.getState()).thenReturn(AMContainerState.IDLE);
+
+ when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
+
+ AMSchedulerEventTALaunchRequest lr =
+ new AMSchedulerEventTALaunchRequest(mockAttemptId, resource, null, mockTaskAttempt, locHint,
+ priority, containerContext, 0, 0, 0);
+ schedulerHandler.taskAllocated(0, mockTaskAttempt, lr, container);
+ assertEquals(2, mockEventHandler.events.size());
+ assertTrue(mockEventHandler.events.get(1) instanceof AMContainerEventAssignTA);
+ AMContainerEventAssignTA assignEvent =
+ (AMContainerEventAssignTA) mockEventHandler.events.get(1);
+ assertEquals(priority, assignEvent.getPriority());
+ assertEquals(mockAttemptId, assignEvent.getTaskAttemptId());
+ }
+
+ @Test (timeout = 5000)
+ public void testTaskBasedAffinity() throws Exception {
+ Configuration conf = new Configuration(false);
+ schedulerHandler.init(conf);
+ schedulerHandler.start();
+
+ TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
+ TezTaskAttemptID taId = mock(TezTaskAttemptID.class);
+ String affVertexName = "srcVertex";
+ int affTaskIndex = 1;
+ TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(affVertexName, affTaskIndex);
+ VertexImpl affVertex = mock(VertexImpl.class);
+ TaskImpl affTask = mock(TaskImpl.class);
+ TaskAttemptImpl affAttempt = mock(TaskAttemptImpl.class);
+ ContainerId affCId = mock(ContainerId.class);
+ when(affVertex.getTotalTasks()).thenReturn(2);
+ when(affVertex.getTask(affTaskIndex)).thenReturn(affTask);
+ when(affTask.getSuccessfulAttempt()).thenReturn(affAttempt);
+ when(affAttempt.getAssignedContainerID()).thenReturn(affCId);
+ when(mockAppContext.getCurrentDAG().getVertex(affVertexName)).thenReturn(affVertex);
+ Resource resource = Resource.newInstance(100, 1);
+ AMSchedulerEventTALaunchRequest event = new AMSchedulerEventTALaunchRequest
+ (taId, resource, null, mockTaskAttempt, locHint, 3, null, 0, 0, 0);
+ schedulerHandler.notify.set(false);
+ schedulerHandler.handle(event);
+ synchronized (schedulerHandler.notify) {
+ while (!schedulerHandler.notify.get()) {
+ schedulerHandler.notify.wait();
+ }
+ }
+
+ // verify mockTaskAttempt affinitized to expected affCId
+ verify(mockTaskScheduler, times(1)).allocateTask(mockTaskAttempt, resource, affCId,
+ Priority.newInstance(3), null, event);
+
+ schedulerHandler.stop();
+ schedulerHandler.close();
+ }
+
+ @Test (timeout = 5000)
+ public void testContainerPreempted() throws IOException {
+ Configuration conf = new Configuration(false);
+ schedulerHandler.init(conf);
+ schedulerHandler.start();
+
+ String diagnostics = "Container preempted by RM.";
+ TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
+ ContainerStatus mockStatus = mock(ContainerStatus.class);
+ ContainerId mockCId = mock(ContainerId.class);
+ AMContainer mockAMContainer = mock(AMContainer.class);
+ when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
+ when(mockAMContainer.getContainerId()).thenReturn(mockCId);
+ when(mockStatus.getContainerId()).thenReturn(mockCId);
+ when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
+ when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.PREEMPTED);
+ schedulerHandler.containerCompleted(0, mockTask, mockStatus);
+ assertEquals(1, mockEventHandler.events.size());
+ Event event = mockEventHandler.events.get(0);
+ assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
+ AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
+ assertEquals(mockCId, completedEvent.getContainerId());
+ assertEquals("Container preempted externally. Container preempted by RM.",
+ completedEvent.getDiagnostics());
+ assertTrue(completedEvent.isPreempted());
+ assertEquals(TaskAttemptTerminationCause.EXTERNAL_PREEMPTION,
+ completedEvent.getTerminationCause());
+ Assert.assertFalse(completedEvent.isDiskFailed());
+
+ schedulerHandler.stop();
+ schedulerHandler.close();
+ }
+
+ @Test (timeout = 5000)
+ public void testContainerInternalPreempted() throws IOException {
+ Configuration conf = new Configuration(false);
+ schedulerHandler.init(conf);
+ schedulerHandler.start();
+
+ AMContainer mockAmContainer = mock(AMContainer.class);
+ when(mockAmContainer.getTaskSchedulerIdentifier()).thenReturn(0);
+ when(mockAmContainer.getContainerLauncherIdentifier()).thenReturn(0);
+ when(mockAmContainer.getTaskCommunicatorIdentifier()).thenReturn(0);
+ ContainerId mockCId = mock(ContainerId.class);
+ verify(mockTaskScheduler, times(0)).deallocateContainer((ContainerId) any());
+ when(mockAMContainerMap.get(mockCId)).thenReturn(mockAmContainer);
+ schedulerHandler.preemptContainer(0, mockCId);
+ verify(mockTaskScheduler, times(1)).deallocateContainer(mockCId);
+ assertEquals(1, mockEventHandler.events.size());
+ Event event = mockEventHandler.events.get(0);
+ assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
+ AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
+ assertEquals(mockCId, completedEvent.getContainerId());
+ assertEquals("Container preempted internally", completedEvent.getDiagnostics());
+ assertTrue(completedEvent.isPreempted());
+ Assert.assertFalse(completedEvent.isDiskFailed());
+ assertEquals(TaskAttemptTerminationCause.INTERNAL_PREEMPTION,
+ completedEvent.getTerminationCause());
+
+ schedulerHandler.stop();
+ schedulerHandler.close();
+ }
+
+ @Test (timeout = 5000)
+ public void testContainerDiskFailed() throws IOException {
+ Configuration conf = new Configuration(false);
+ schedulerHandler.init(conf);
+ schedulerHandler.start();
+
+ String diagnostics = "NM disk failed.";
+ TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
+ ContainerStatus mockStatus = mock(ContainerStatus.class);
+ ContainerId mockCId = mock(ContainerId.class);
+ AMContainer mockAMContainer = mock(AMContainer.class);
+ when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
+ when(mockAMContainer.getContainerId()).thenReturn(mockCId);
+ when(mockStatus.getContainerId()).thenReturn(mockCId);
+ when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
+ when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.DISKS_FAILED);
+ schedulerHandler.containerCompleted(0, mockTask, mockStatus);
+ assertEquals(1, mockEventHandler.events.size());
+ Event event = mockEventHandler.events.get(0);
+ assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
+ AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
+ assertEquals(mockCId, completedEvent.getContainerId());
+ assertEquals("Container disk failed. NM disk failed.",
+ completedEvent.getDiagnostics());
+ Assert.assertFalse(completedEvent.isPreempted());
+ assertTrue(completedEvent.isDiskFailed());
+ assertEquals(TaskAttemptTerminationCause.NODE_DISK_ERROR,
+ completedEvent.getTerminationCause());
+
+ schedulerHandler.stop();
+ schedulerHandler.close();
+ }
+
+ @Test (timeout = 5000)
+ public void testContainerExceededPMem() throws IOException {
+ Configuration conf = new Configuration(false);
+ schedulerHandler.init(conf);
+ schedulerHandler.start();
+
+ String diagnostics = "Exceeded Physical Memory";
+ TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
+ ContainerStatus mockStatus = mock(ContainerStatus.class);
+ ContainerId mockCId = mock(ContainerId.class);
+ AMContainer mockAMContainer = mock(AMContainer.class);
+ when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
+ when(mockAMContainer.getContainerId()).thenReturn(mockCId);
+ when(mockStatus.getContainerId()).thenReturn(mockCId);
+ when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
+ // use -104 rather than ContainerExitStatus.KILLED_EXCEEDED_PMEM because
+ // ContainerExitStatus.KILLED_EXCEEDED_PMEM is only available after hadoop-2.5
+ when(mockStatus.getExitStatus()).thenReturn(-104);
+ schedulerHandler.containerCompleted(0, mockTask, mockStatus);
+ assertEquals(1, mockEventHandler.events.size());
+ Event event = mockEventHandler.events.get(0);
+ assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
+ AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
+ assertEquals(mockCId, completedEvent.getContainerId());
+ assertEquals("Container failed, exitCode=-104. Exceeded Physical Memory",
+ completedEvent.getDiagnostics());
+ Assert.assertFalse(completedEvent.isPreempted());
+ Assert.assertFalse(completedEvent.isDiskFailed());
+ assertEquals(TaskAttemptTerminationCause.CONTAINER_EXITED,
+ completedEvent.getTerminationCause());
+
+ schedulerHandler.stop();
+ schedulerHandler.close();
+ }
+
+ @Test (timeout = 5000)
+ public void testHistoryUrlConf() throws Exception {
+ Configuration conf = schedulerHandler.appContext.getAMConf();
+
+ // ensure history url is empty when timeline server is not the logging class
+ conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9999");
+ assertTrue("".equals(schedulerHandler.getHistoryUrl()));
+
+ // ensure expansion of url happens
+ conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
+ "org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService");
+ final ApplicationId mockApplicationId = mock(ApplicationId.class);
+ doReturn("TEST_APP_ID").when(mockApplicationId).toString();
+ doReturn(mockApplicationId).when(mockAppContext).getApplicationID();
+ assertTrue("http://ui-host:9999/#/tez-app/TEST_APP_ID"
+ .equals(schedulerHandler.getHistoryUrl()));
+
+ // ensure the trailing / in history url is handled
+ conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9998/");
+ assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
+ .equals(schedulerHandler.getHistoryUrl()));
+
+ // ensure missing scheme in history url is handled
+ conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/");
+ Assert.assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
+ .equals(schedulerHandler.getHistoryUrl()));
+
+ // handle bad template ex without begining /
+ conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
+ "__HISTORY_URL_BASE__#/somepath");
+ assertTrue("http://ui-host:9998/#/somepath"
+ .equals(schedulerHandler.getHistoryUrl()));
+
+ conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
+ "__HISTORY_URL_BASE__?viewPath=tez-app/__APPLICATION_ID__");
+ conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://localhost/ui/tez");
+ assertTrue("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID"
+ .equals(schedulerHandler.getHistoryUrl()));
+
+ }
+
+ @Test(timeout = 5000)
+ public void testNoSchedulerSpecified() throws IOException {
+ try {
+ new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
+ mockSigMatcher, mockWebUIService, null, false);
+ fail("Expecting an IllegalStateException with no schedulers specified");
+ } catch (IllegalArgumentException e) {
+ }
+ }
+
+ // Verified via statics
+ @Test(timeout = 5000)
+ public void testCustomTaskSchedulerSetup() throws IOException {
+ Configuration conf = new Configuration(false);
+ conf.set("testkey", "testval");
+ UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
+
+ String customSchedulerName = "fakeScheduler";
+ List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
+ ByteBuffer bb = ByteBuffer.allocate(4);
+ bb.putInt(0, 3);
+ UserPayload userPayload = UserPayload.create(bb);
+ taskSchedulers.add(
+ new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName())
+ .setUserPayload(userPayload));
+ taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+ .setUserPayload(defaultPayload));
+
+ TSEHForMultipleSchedulersTest tseh =
+ new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
+ mockSigMatcher, mockWebUIService, taskSchedulers, false);
+
+ tseh.init(conf);
+ tseh.start();
+
+ // Verify that the YARN task scheduler is installed by default
+ assertTrue(tseh.getYarnSchedulerCreated());
+ assertFalse(tseh.getUberSchedulerCreated());
+ assertEquals(2, tseh.getNumCreateInvocations());
+
+ // Verify the order of the schedulers
+ assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
+ assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));
+
+ // Verify the payload setup for the custom task scheduler
+ assertNotNull(tseh.getTaskSchedulerContext(0));
+ assertEquals(bb, tseh.getTaskSchedulerContext(0).getInitialUserPayload().getPayload());
+
+ // Verify the payload on the yarn scheduler
+ assertNotNull(tseh.getTaskSchedulerContext(1));
+ Configuration parsed = TezUtils.createConfFromUserPayload(tseh.getTaskSchedulerContext(1).getInitialUserPayload());
+ assertEquals("testval", parsed.get("testkey"));
+ }
+
+ @Test(timeout = 5000)
+ public void testTaskSchedulerRouting() throws Exception {
+ Configuration conf = new Configuration(false);
+ UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
+
+ String customSchedulerName = "fakeScheduler";
+ List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
+ ByteBuffer bb = ByteBuffer.allocate(4);
+ bb.putInt(0, 3);
+ UserPayload userPayload = UserPayload.create(bb);
+ taskSchedulers.add(
+ new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName())
+ .setUserPayload(userPayload));
+ taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+ .setUserPayload(defaultPayload));
+
+ TSEHForMultipleSchedulersTest tseh =
+ new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
+ mockSigMatcher, mockWebUIService, taskSchedulers, false);
+
+ tseh.init(conf);
+ tseh.start();
+
+ // Verify that the YARN task scheduler is installed by default
+ assertTrue(tseh.getYarnSchedulerCreated());
+ assertFalse(tseh.getUberSchedulerCreated());
+ assertEquals(2, tseh.getNumCreateInvocations());
+
+ // Verify the order of the schedulers
+ assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
+ assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));
+
+ verify(tseh.getTestTaskScheduler(0)).initialize();
+ verify(tseh.getTestTaskScheduler(0)).start();
+
+ ApplicationId appId = ApplicationId.newInstance(1000, 1);
+ TezDAGID dagId = TezDAGID.getInstance(appId, 1);
+ TezVertexID vertexID = TezVertexID.getInstance(dagId, 1);
+ TezTaskID taskId1 = TezTaskID.getInstance(vertexID, 1);
+ TezTaskAttemptID attemptId11 = TezTaskAttemptID.getInstance(taskId1, 1);
+ TezTaskID taskId2 = TezTaskID.getInstance(vertexID, 2);
+ TezTaskAttemptID attemptId21 = TezTaskAttemptID.getInstance(taskId2, 1);
+
+ Resource resource = Resource.newInstance(1024, 1);
+
+ TaskAttempt mockTaskAttempt1 = mock(TaskAttempt.class);
+ TaskAttempt mockTaskAttempt2 = mock(TaskAttempt.class);
+
+ AMSchedulerEventTALaunchRequest launchRequest1 =
+ new AMSchedulerEventTALaunchRequest(attemptId11, resource, mock(TaskSpec.class),
+ mockTaskAttempt1, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 0, 0,
+ 0);
+
+ tseh.handle(launchRequest1);
+
+ verify(tseh.getTestTaskScheduler(0)).allocateTask(eq(mockTaskAttempt1), eq(resource),
+ any(String[].class), any(String[].class), any(Priority.class), any(Object.class),
+ eq(launchRequest1));
+
+ AMSchedulerEventTALaunchRequest launchRequest2 =
+ new AMSchedulerEventTALaunchRequest(attemptId21, resource, mock(TaskSpec.class),
+ mockTaskAttempt2, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 1, 0,
+ 0);
+ tseh.handle(launchRequest2);
+ verify(tseh.getTestTaskScheduler(1)).allocateTask(eq(mockTaskAttempt2), eq(resource),
+ any(String[].class), any(String[].class), any(Priority.class), any(Object.class),
+ eq(launchRequest2));
+ }
+
+ private static class TSEHForMultipleSchedulersTest extends TaskSchedulerManager {
+
+ private final TaskScheduler yarnTaskScheduler;
+ private final TaskScheduler uberTaskScheduler;
+ private final AtomicBoolean uberSchedulerCreated = new AtomicBoolean(false);
+ private final AtomicBoolean yarnSchedulerCreated = new AtomicBoolean(false);
+ private final AtomicInteger numCreateInvocations = new AtomicInteger(0);
+ private final Set<Integer> seenSchedulers = new HashSet<>();
+ private final List<TaskSchedulerContext> taskSchedulerContexts = new LinkedList<>();
+ private final List<String> taskSchedulerNames = new LinkedList<>();
+ private final List<TaskScheduler> testTaskSchedulers = new LinkedList<>();
+
+ public TSEHForMultipleSchedulersTest(AppContext appContext,
+ DAGClientServer clientService,
+ EventHandler eventHandler,
+ ContainerSignatureMatcher containerSignatureMatcher,
+ WebUIService webUI,
+ List<NamedEntityDescriptor> schedulerDescriptors,
+ boolean isPureLocalMode) {
+ super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI,
+ schedulerDescriptors, isPureLocalMode);
+ yarnTaskScheduler = mock(TaskScheduler.class);
+ uberTaskScheduler = mock(TaskScheduler.class);
+ }
+
+ @Override
+ TaskScheduler createTaskScheduler(String host, int port, String trackingUrl,
+ AppContext appContext,
+ NamedEntityDescriptor taskSchedulerDescriptor,
+ long customAppIdIdentifier,
+ int schedulerId) {
+
+ numCreateInvocations.incrementAndGet();
+ boolean added = seenSchedulers.add(schedulerId);
+ assertTrue("Cannot add multiple schedulers with the same schedulerId", added);
+ taskSchedulerNames.add(taskSchedulerDescriptor.getEntityName());
+ return super.createTaskScheduler(host, port, trackingUrl, appContext, taskSchedulerDescriptor,
+ customAppIdIdentifier, schedulerId);
+ }
+
+ @Override
+ TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
+ // Avoid wrapping in threads
+ return rawContext;
+ }
+
+ @Override
+ TaskScheduler createYarnTaskScheduler(TaskSchedulerContext taskSchedulerContext, int schedulerId) {
+ taskSchedulerContexts.add(taskSchedulerContext);
+ testTaskSchedulers.add(yarnTaskScheduler);
+ yarnSchedulerCreated.set(true);
+ return yarnTaskScheduler;
+ }
+
+ @Override
+ TaskScheduler createUberTaskScheduler(TaskSchedulerContext taskSchedulerContext, int schedulerId) {
+ taskSchedulerContexts.add(taskSchedulerContext);
+ uberSchedulerCreated.set(true);
+ testTaskSchedulers.add(yarnTaskScheduler);
+ return uberTaskScheduler;
+ }
+
+ @Override
+ TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext,
+ NamedEntityDescriptor taskSchedulerDescriptor, int schedulerId) {
+ taskSchedulerContexts.add(taskSchedulerContext);
+ TaskScheduler taskScheduler = spy(super.createCustomTaskScheduler(taskSchedulerContext, taskSchedulerDescriptor, schedulerId));
+ testTaskSchedulers.add(taskScheduler);
+ return taskScheduler;
+ }
+
+ @Override
+ // Inline handling of events.
+ public void handle(AMSchedulerEvent event) {
+ handleEvent(event);
+ }
+
+ public boolean getUberSchedulerCreated() {
+ return uberSchedulerCreated.get();
+ }
+
+ public boolean getYarnSchedulerCreated() {
+ return yarnSchedulerCreated.get();
+ }
+
+ public int getNumCreateInvocations() {
+ return numCreateInvocations.get();
+ }
+
+ public TaskSchedulerContext getTaskSchedulerContext(int schedulerId) {
+ return taskSchedulerContexts.get(schedulerId);
+ }
+
+ public String getTaskSchedulerName(int schedulerId) {
+ return taskSchedulerNames.get(schedulerId);
+ }
+
+ public TaskScheduler getTestTaskScheduler(int schedulerId) {
+ return testTaskSchedulers.get(schedulerId);
+ }
+ }
+
+ public static class FakeTaskScheduler extends TaskScheduler {
+
+ public FakeTaskScheduler(
+ TaskSchedulerContext taskSchedulerContext) {
+ super(taskSchedulerContext);
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ return null;
+ }
+
+ @Override
+ public int getClusterNodeCount() {
+ return 0;
+ }
+
+ @Override
+ public void dagComplete() {
+
+ }
+
+ @Override
+ public Resource getTotalResources() {
+ return null;
+ }
+
+ @Override
+ public void blacklistNode(NodeId nodeId) {
+
+ }
+
+ @Override
+ public void unblacklistNode(NodeId nodeId) {
+
+ }
+
+ @Override
+ public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks,
+ Priority priority, Object containerSignature, Object clientCookie) {
+
+ }
+
+ @Override
+ public void allocateTask(Object task, Resource capability, ContainerId containerId,
+ Priority priority, Object containerSignature, Object clientCookie) {
+
+ }
+
+ @Override
+ public boolean deallocateTask(Object task, boolean taskSucceeded,
+ TaskAttemptEndReason endReason,
+ String diagnostics) {
+ return false;
+ }
+
+ @Override
+ public Object deallocateContainer(ContainerId containerId) {
+ return null;
+ }
+
+ @Override
+ public void setShouldUnregister() {
+
+ }
+
+ @Override
+ public boolean hasUnregistered() {
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index 13fa4c5..cc88f0d 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -69,14 +69,14 @@ import org.apache.tez.dag.api.TaskCommunicator;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerHeartbeatHandler;
import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminatedBySystem;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminating;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventNodeFailed;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
import org.apache.tez.dag.app.rm.AMSchedulerEventType;
-import org.apache.tez.dag.app.rm.NMCommunicatorEventType;
+import org.apache.tez.dag.app.rm.ContainerLauncherEventType;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.HistoryEventHandler;
import org.apache.tez.dag.records.TaskAttemptTerminationCause;
@@ -288,7 +288,7 @@ public class TestAMContainer {
// Event to NM to stop the container.
wc.verifyCountAndGetOutgoingEvents(1);
assertTrue(wc.verifyCountAndGetOutgoingEvents(1).get(0).getType() ==
- NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+ ContainerLauncherEventType.CONTAINER_STOP_REQUEST);
wc.nmStopSent();
wc.verifyState(AMContainerState.STOPPING);
@@ -323,7 +323,7 @@ public class TestAMContainer {
// Event to NM to stop the container.
wc.verifyCountAndGetOutgoingEvents(1);
assertTrue(wc.verifyCountAndGetOutgoingEvents(1).get(0).getType() ==
- NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+ ContainerLauncherEventType.CONTAINER_STOP_REQUEST);
wc.nmStopFailed();
wc.verifyState(AMContainerState.STOPPING);
@@ -366,7 +366,7 @@ public class TestAMContainer {
// 1 for NM stop request. 2 TERMINATING to TaskAttempt.
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
- NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
+ ContainerLauncherEventType.CONTAINER_STOP_REQUEST,
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
TaskAttemptEventType.TA_CONTAINER_TERMINATING);
assertTrue(wc.amContainer.isInErrorState());
@@ -405,7 +405,7 @@ public class TestAMContainer {
// 1 for NM stop request. 2 TERMINATING to TaskAttempt.
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
- NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
+ ContainerLauncherEventType.CONTAINER_STOP_REQUEST,
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
TaskAttemptEventType.TA_CONTAINER_TERMINATING);
assertTrue(wc.amContainer.isInErrorState());
@@ -443,7 +443,7 @@ public class TestAMContainer {
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
- NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+ ContainerLauncherEventType.CONTAINER_STOP_REQUEST);
// TODO Should this be an RM DE-ALLOCATE instead ?
wc.containerCompleted();
@@ -478,7 +478,7 @@ public class TestAMContainer {
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
- NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+ ContainerLauncherEventType.CONTAINER_STOP_REQUEST);
// TODO Should this be an RM DE-ALLOCATE instead ?
wc.containerCompleted();
@@ -1194,7 +1194,7 @@ public class TestAMContainer {
Priority priority;
Container container;
ContainerHeartbeatHandler chh;
- TaskAttemptListener tal;
+ TaskCommunicatorManagerInterface tal;
@SuppressWarnings("rawtypes")
EventHandler eventHandler;
@@ -1226,7 +1226,7 @@ public class TestAMContainer {
chh = mock(ContainerHeartbeatHandler.class);
- tal = mock(TaskAttemptListener.class);
+ tal = mock(TaskCommunicatorManagerInterface.class);
TaskCommunicator taskComm = mock(TaskCommunicator.class);
doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
doReturn(taskComm).when(tal).getTaskCommunicator(0);
@@ -1440,7 +1440,7 @@ public class TestAMContainer {
return lr;
}
- private void verifyUnregisterRunningContainer(TaskAttemptListener tal, ContainerId containerId,
+ private void verifyUnregisterRunningContainer(TaskCommunicatorManagerInterface tal, ContainerId containerId,
int taskCommId,
ContainerEndReason containerEndReason,
String diagContains) {
@@ -1455,7 +1455,7 @@ public class TestAMContainer {
}
}
- private void verifyUnregisterTaskAttempt(TaskAttemptListener tal, TezTaskAttemptID taId,
+ private void verifyUnregisterTaskAttempt(TaskCommunicatorManagerInterface tal, TezTaskAttemptID taId,
int taskCommId, TaskAttemptEndReason endReason,
String diagContains) {
ArgumentCaptor<String> argumentCaptor = ArgumentCaptor.forClass(String.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
index dee4541..0230bb5 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.Token;
import org.apache.tez.dag.api.TaskCommunicator;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerHeartbeatHandler;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
public class TestAMContainerMap {
@@ -42,8 +42,8 @@ public class TestAMContainerMap {
return mock(ContainerHeartbeatHandler.class);
}
- private TaskAttemptListener mockTaskAttemptListener() {
- TaskAttemptListener tal = mock(TaskAttemptListener.class);
+ private TaskCommunicatorManagerInterface mockTaskAttemptListener() {
+ TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
TaskCommunicator taskComm = mock(TaskCommunicator.class);
doReturn(new InetSocketAddress("localhost", 21000)).when(taskComm).getAddress();
doReturn(taskComm).when(tal).getTaskCommunicator(0);
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
index def80da..143fcbf 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
@@ -44,7 +44,7 @@ import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.rm.AMSchedulerEventNodeBlacklistUpdate;
import org.apache.tez.dag.app.rm.AMSchedulerEventType;
-import org.apache.tez.dag.app.rm.TaskSchedulerEventHandler;
+import org.apache.tez.dag.app.rm.TaskSchedulerManager;
import org.apache.tez.dag.app.rm.container.AMContainerEventNodeFailed;
import org.apache.tez.dag.app.rm.container.AMContainerEventType;
import org.apache.tez.dag.app.rm.container.AMContainerMap;
@@ -186,11 +186,11 @@ public class TestAMNodeTracker {
AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
doReturn(amNodeTracker).when(appContext).getNodeTracker();
AMContainerMap amContainerMap = mock(AMContainerMap.class);
- TaskSchedulerEventHandler taskSchedulerEventHandler =
- mock(TaskSchedulerEventHandler.class);
+ TaskSchedulerManager taskSchedulerManager =
+ mock(TaskSchedulerManager.class);
dispatcher.register(AMNodeEventType.class, amNodeTracker);
dispatcher.register(AMContainerEventType.class, amContainerMap);
- dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+ dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
amNodeTracker.init(conf);
amNodeTracker.start();
@@ -209,11 +209,11 @@ public class TestAMNodeTracker {
AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
doReturn(amNodeTracker).when(appContext).getNodeTracker();
AMContainerMap amContainerMap = mock(AMContainerMap.class);
- TaskSchedulerEventHandler taskSchedulerEventHandler =
- mock(TaskSchedulerEventHandler.class);
+ TaskSchedulerManager taskSchedulerManager =
+ mock(TaskSchedulerManager.class);
dispatcher.register(AMNodeEventType.class, amNodeTracker);
dispatcher.register(AMContainerEventType.class, amContainerMap);
- dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+ dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
amNodeTracker.init(conf);
amNodeTracker.start();
@@ -232,11 +232,11 @@ public class TestAMNodeTracker {
AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
doReturn(amNodeTracker).when(appContext).getNodeTracker();
AMContainerMap amContainerMap = mock(AMContainerMap.class);
- TaskSchedulerEventHandler taskSchedulerEventHandler =
- mock(TaskSchedulerEventHandler.class);
+ TaskSchedulerManager taskSchedulerManager =
+ mock(TaskSchedulerManager.class);
dispatcher.register(AMNodeEventType.class, amNodeTracker);
dispatcher.register(AMContainerEventType.class, amContainerMap);
- dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+ dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
amNodeTracker.init(conf);
amNodeTracker.start();
@@ -260,11 +260,11 @@ public class TestAMNodeTracker {
AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
doReturn(amNodeTracker).when(appContext).getNodeTracker();
AMContainerMap amContainerMap = mock(AMContainerMap.class);
- TaskSchedulerEventHandler taskSchedulerEventHandler =
- mock(TaskSchedulerEventHandler.class);
+ TaskSchedulerManager taskSchedulerManager =
+ mock(TaskSchedulerManager.class);
dispatcher.register(AMNodeEventType.class, amNodeTracker);
dispatcher.register(AMContainerEventType.class, amContainerMap);
- dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+ dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
amNodeTracker.init(conf);
amNodeTracker.start();
try {
@@ -283,11 +283,11 @@ public class TestAMNodeTracker {
AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
doReturn(amNodeTracker).when(appContext).getNodeTracker();
AMContainerMap amContainerMap = mock(AMContainerMap.class);
- TaskSchedulerEventHandler taskSchedulerEventHandler =
- mock(TaskSchedulerEventHandler.class);
+ TaskSchedulerManager taskSchedulerManager =
+ mock(TaskSchedulerManager.class);
dispatcher.register(AMNodeEventType.class, amNodeTracker);
dispatcher.register(AMContainerEventType.class, amContainerMap);
- dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+ dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
amNodeTracker.init(conf);
amNodeTracker.start();
try {
@@ -306,11 +306,11 @@ public class TestAMNodeTracker {
AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
doReturn(amNodeTracker).when(appContext).getNodeTracker();
AMContainerMap amContainerMap = mock(AMContainerMap.class);
- TaskSchedulerEventHandler taskSchedulerEventHandler =
- mock(TaskSchedulerEventHandler.class);
+ TaskSchedulerManager taskSchedulerManager =
+ mock(TaskSchedulerManager.class);
dispatcher.register(AMNodeEventType.class, amNodeTracker);
dispatcher.register(AMContainerEventType.class, amContainerMap);
- dispatcher.register(AMSchedulerEventType.class, taskSchedulerEventHandler);
+ dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
amNodeTracker.init(conf);
amNodeTracker.start();
try {
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
----------------------------------------------------------------------
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
index 3b4c768..fb4c08f 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java
@@ -60,7 +60,8 @@ import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.runtime.api.impl.TaskSpec;
import org.apache.tez.runtime.task.TaskReporter;
-import org.apache.tez.runtime.task.TezTaskRunner;
+import org.apache.tez.runtime.task.TaskRunner2Result;
+import org.apache.tez.runtime.task.TezTaskRunner2;
import org.apache.tez.service.ContainerRunner;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.runtime.api.ExecutionContext;
@@ -378,7 +379,7 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
private final Credentials credentials;
private final long memoryAvailable;
private final ListeningExecutorService executor;
- private volatile TezTaskRunner taskRunner;
+ private volatile TezTaskRunner2 taskRunner;
private volatile TaskReporter taskReporter;
private TezTaskUmbilicalProtocol umbilical;
@@ -443,7 +444,7 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
new AtomicLong(0),
request.getContainerIdString());
- taskRunner = new TezTaskRunner(conf, taskUgi, localDirs,
+ taskRunner = new TezTaskRunner2(conf, taskUgi, localDirs,
ProtoConverters.getTaskSpecfromProto(request.getTaskSpec()),
request.getAppAttemptNumber(),
serviceConsumerMetadata, envMap, startedInputsMap, taskReporter, executor, objectRegistry,
@@ -452,18 +453,20 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun
boolean shouldDie;
try {
- shouldDie = !taskRunner.run();
+ TaskRunner2Result result = taskRunner.run();
+ LOG.info("TaskRunner2Result: {}", result);
+ shouldDie = result.isContainerShutdownRequested();
if (shouldDie) {
LOG.info("Got a shouldDie notification via heartbeats. Shutting down");
return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null,
"Asked to die by the AM");
}
- } catch (IOException e) {
- return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE,
- e, "TaskExecutionFailure: " + e.getMessage());
- } catch (TezException e) {
- return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE,
- e, "TaskExecutionFailure: " + e.getMessage());
+ if (result.getError() != null) {
+ Throwable e = result.getError();
+ return new ContainerExecutionResult(
+ ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE,
+ e, "TaskExecutionFailure: " + e.getMessage());
+ }
} finally {
FileSystem.closeAllForUGI(taskUgi);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
deleted file mode 100644
index aebf6a9..0000000
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
+++ /dev/null
@@ -1,451 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.runtime.task;
-
-import java.io.IOException;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.nio.ByteBuffer;
-import java.security.PrivilegedExceptionAction;
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import com.google.common.base.Throwables;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSError;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.tez.common.CallableWithNdc;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.runtime.LogicalIOProcessorRuntimeTask;
-import org.apache.tez.runtime.RuntimeTask;
-import org.apache.tez.runtime.api.ExecutionContext;
-import org.apache.tez.runtime.api.ObjectRegistry;
-import org.apache.tez.runtime.api.impl.EventMetaData;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.runtime.api.impl.TezEvent;
-import org.apache.tez.runtime.api.impl.TezUmbilical;
-import org.apache.tez.runtime.internals.api.TaskReporterInterface;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Multimap;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-
-public class TezTaskRunner implements TezUmbilical, ErrorReporter {
-
- private static final Logger LOG = LoggerFactory.getLogger(TezTaskRunner.class);
-
- private final Configuration tezConf;
- private final LogicalIOProcessorRuntimeTask task;
- private final UserGroupInformation ugi;
-
- private final TaskReporterInterface taskReporter;
- private final ListeningExecutorService executor;
- private volatile ListenableFuture<Void> taskFuture;
- private volatile Thread waitingThread;
- private volatile Thread taskRunner;
- private volatile Throwable firstException;
-
- // Effectively a duplicate check, since hadFatalError does the same thing.
- private final AtomicBoolean fatalErrorSent = new AtomicBoolean(false);
- private final AtomicBoolean taskRunning;
- private final AtomicBoolean shutdownRequested = new AtomicBoolean(false);
-
- public TezTaskRunner(Configuration tezConf, UserGroupInformation ugi, String[] localDirs,
- TaskSpec taskSpec, int appAttemptNumber,
- Map<String, ByteBuffer> serviceConsumerMetadata, Map<String, String> serviceProviderEnvMap,
- Multimap<String, String> startedInputsMap, TaskReporterInterface taskReporter,
- ListeningExecutorService executor, ObjectRegistry objectRegistry, String pid,
- ExecutionContext executionContext, long memAvailable)
- throws IOException {
- this.tezConf = tezConf;
- this.ugi = ugi;
- this.taskReporter = taskReporter;
- this.executor = executor;
- task = new LogicalIOProcessorRuntimeTask(taskSpec, appAttemptNumber, tezConf, localDirs, this,
- serviceConsumerMetadata, serviceProviderEnvMap, startedInputsMap, objectRegistry, pid,
- executionContext, memAvailable);
- taskRunning = new AtomicBoolean(false);
- }
-
- /**
- * @return false if a shutdown message was received during task execution
- * @throws TezException
- * @throws IOException
- */
- public boolean run() throws InterruptedException, IOException, TezException {
- waitingThread = Thread.currentThread();
- taskRunning.set(true);
- taskReporter.registerTask(task, this);
- TaskRunnerCallable callable = new TaskRunnerCallable();
- Throwable failureCause = null;
- if (!Thread.currentThread().isInterrupted()) {
- taskFuture = executor.submit(callable);
- } else {
- taskReporter.unregisterTask(task.getTaskAttemptID());
- return isShutdownRequested();
- }
- try {
- taskFuture.get();
-
- // Task could signal a fatal error and return control, or a failure while registering success.
- failureCause = firstException;
-
- } catch (InterruptedException e) {
- LOG.info("Interrupted while waiting for task to complete. Interrupting task");
- taskFuture.cancel(true);
- if (shutdownRequested.get()) {
- LOG.info("Shutdown requested... returning");
- return false;
- }
- if (firstException != null) {
- failureCause = firstException;
- } else {
- // Interrupted for some other reason.
- failureCause = e;
- }
- } catch (ExecutionException e) {
- // Exception thrown by the run() method itself.
- Throwable cause = e.getCause();
- if (cause instanceof FSError) {
- // Not immediately fatal, this is an error reported by Hadoop FileSystem
- failureCause = cause;
- } else if (cause instanceof Error) {
- LOG.error("Exception of type Error.", cause);
- sendFailure(cause, "Fatal Error cause TezChild exit.");
- throw new TezException("Fatal Error cause TezChild exit.", cause);
- } else {
- failureCause = cause;
- }
- } finally {
- // Clear the interrupted status of the blocking thread, in case it is set after the
- // InterruptedException was invoked.
- taskReporter.unregisterTask(task.getTaskAttemptID());
- Thread.interrupted();
- }
-
- if (failureCause != null) {
- if (failureCause instanceof FSError) {
- // Not immediately fatal, this is an error reported by Hadoop FileSystem
- LOG.info("Encountered an FSError while executing task: " + task.getTaskAttemptID(),
- failureCause);
- throw (FSError) failureCause;
- } else if (failureCause instanceof Error) {
- LOG.error("Exception of type Error.", failureCause);
- sendFailure(failureCause, "Fatal error cause TezChild exit.");
- throw new TezException("Fatal error cause TezChild exit.", failureCause);
- } else {
- if (failureCause instanceof IOException) {
- throw (IOException) failureCause;
- } else if (failureCause instanceof TezException) {
- throw (TezException) failureCause;
- } else if (failureCause instanceof InterruptedException) {
- throw (InterruptedException) failureCause;
- } else {
- throw new TezException(failureCause);
- }
- }
- }
- return isShutdownRequested();
- }
-
- private boolean isShutdownRequested() {
- if (shutdownRequested.get()) {
- LOG.info("Shutdown requested... returning");
- return false;
- }
- return true;
- }
-
- private class TaskRunnerCallable extends CallableWithNdc<Void> {
- @Override
- protected Void callInternal() throws Exception {
- try {
- return ugi.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- try {
- taskRunner = Thread.currentThread();
- LOG.info("Initializing task" + ", taskAttemptId=" + task.getTaskAttemptID());
- task.initialize();
- if (!Thread.currentThread().isInterrupted() && firstException == null) {
- LOG.info("Running task, taskAttemptId=" + task.getTaskAttemptID());
- task.run();
- maybeInterruptWaitingThread();
-
- LOG.info("Closing task, taskAttemptId=" + task.getTaskAttemptID());
- task.close();
- task.setFrameworkCounters();
- }
- LOG.info("Task completed, taskAttemptId=" + task.getTaskAttemptID()
- + ", fatalErrorOccurred=" + (firstException != null));
- if (firstException == null) {
- try {
- taskReporter.taskSucceeded(task.getTaskAttemptID());
- } catch (IOException e) {
- LOG.warn("Heartbeat failure caused by communication failure", e);
- maybeRegisterFirstException(e);
- // Falling off, since the runner thread checks for the registered exception.
- } catch (TezException e) {
- LOG.warn("Heartbeat failure reported by AM", e);
- maybeRegisterFirstException(e);
- // Falling off, since the runner thread checks for the registered exception.
- }
- }
- return null;
- } catch (Throwable cause) {
- if (Thread.currentThread().isInterrupted()) {
- LOG.info("TaskRunnerCallable interrupted=" + Thread.currentThread().isInterrupted()
- + ", shutdownRequest=" + shutdownRequested.get());
- Thread.currentThread().interrupt();
- return null;
- }
- if (cause instanceof FSError) {
- // Not immediately fatal, this is an error reported by Hadoop FileSystem
- maybeRegisterFirstException(cause);
- LOG.info("Encountered an FSError while executing task: " + task.getTaskAttemptID(),
- cause);
- try {
- sendFailure(cause, "FS Error in Child JVM");
- } catch (Exception ignored) {
- // Ignored since another cause is already known
- LOG.info(
- "Ignoring the following exception since a previous exception is already registered",
- ignored.getClass().getName());
- if (LOG.isTraceEnabled()) {
- LOG.trace("Ignored exception is", ignored);
- }
- }
- throw (FSError) cause;
- } else if (cause instanceof Error) {
- LOG.error("Exception of type Error.", cause);
- sendFailure(cause, "Fatal Error cause TezChild exit.");
- throw new TezException("Fatal Error cause TezChild exit.", cause);
- } else {
- if (cause instanceof UndeclaredThrowableException) {
- cause = ((UndeclaredThrowableException) cause).getCause();
- }
- maybeRegisterFirstException(cause);
- LOG.info("Encountered an error while executing task: " + task.getTaskAttemptID(),
- cause);
- try {
- sendFailure(cause, "Failure while running task");
- } catch (Exception ignored) {
- // Ignored since another cause is already known
- LOG.info(
- "Ignoring the following exception since a previous exception is already registered",
- ignored.getClass().getName());
- if (LOG.isTraceEnabled()) {
- LOG.trace("Ignored exception is", ignored);
- }
- }
- if (cause instanceof IOException) {
- throw (IOException) cause;
- } else if (cause instanceof TezException) {
- throw (TezException) cause;
- } else {
- throw new TezException(cause);
- }
- }
- } finally {
- task.cleanup();
- }
- }
- });
- } finally {
- taskRunning.set(false);
- }
- }
-
- private void maybeInterruptWaitingThread() {
- /**
- * Possible that the processor is swallowing InterruptException of taskRunner.interrupt().
- * In such case, interrupt the waitingThread based on the shutdownRequested flag, so that
- * entire task gets cancelled.
- */
- if (shutdownRequested.get()) {
- waitingThread.interrupt();
- }
- }
- }
-
- // should wait until all messages are sent to AM before TezChild shutdown
- // if this method become async in future
- private void sendFailure(Throwable t, String message) throws IOException, TezException {
- if (!fatalErrorSent.getAndSet(true)) {
- task.setFatalError(t, message);
- task.setFrameworkCounters();
- try {
- taskReporter.taskFailed(task.getTaskAttemptID(), t, message, null);
- } catch (IOException e) {
- // A failure reason already exists, Comm error just logged.
- LOG.warn("Heartbeat failure caused by communication failure", e);
- throw e;
- } catch (TezException e) {
- // A failure reason already exists, Comm error just logged.
- LOG.warn("Heartbeat failure reported by AM", e);
- throw e;
- }
- } else {
- LOG.warn("Ignoring fatal error since another error has already been reported", t);
- }
- }
-
- @Override
- public void addEvents(Collection<TezEvent> events) {
- if (taskRunning.get()) {
- taskReporter.addEvents(task.getTaskAttemptID(), events);
- }
- }
-
- @Override
- public synchronized void signalFatalError(TezTaskAttemptID taskAttemptID, Throwable t,
- String message, EventMetaData sourceInfo) {
- // This can be called before a task throws an exception or after it.
- // If called before a task throws an exception
- // - ensure a heartbeat is sent with the diagnostics, and sent only once.
- // - interrupt the waiting thread, and make it throw the reported error.
- // If called after a task throws an exception, the waiting task has already returned, no point
- // interrupting it.
- // This case can be effectively ignored (log), as long as the run() method ends up throwing the
- // exception.
- //
- //
- if (!fatalErrorSent.getAndSet(true)) {
- maybeRegisterFirstException(t);
- try {
- taskReporter.taskFailed(taskAttemptID, t, getTaskDiagnosticsString(t, message), sourceInfo);
- } catch (IOException e) {
- // HeartbeatFailed. Don't need to propagate the heartbeat exception since a task exception
- // occurred earlier.
- LOG.warn("Heartbeat failure caused by communication failure", e);
- } catch (TezException e) {
- // HeartbeatFailed. Don't need to propagate the heartbeat exception since a task exception
- // occurred earlier.
- LOG.warn("Heartbeat failure reported by AM", e);
- } finally {
- // Wake up the waiting thread so that it can return control
- waitingThread.interrupt();
- }
- }
- }
-
- @Override
- public boolean canCommit(TezTaskAttemptID taskAttemptID) {
- if (taskRunning.get()) {
- try {
- return taskReporter.canCommit(taskAttemptID);
- } catch (IOException e) {
- LOG.warn("Communication failure while trying to commit", e);
- maybeRegisterFirstException(e);
- waitingThread.interrupt();
- // Not informing the task since it will be interrupted.
- // TODO: Should this be sent to the task as well, current Processors, etc do not handle
- // interrupts very well.
- return false;
- }
- } else {
- return false;
- }
- }
-
- @Override
- public synchronized void reportError(Throwable t) {
- if (taskRunning.get()) {
- LOG.error("TaskReporter reported error", t);
- maybeRegisterFirstException(t);
- waitingThread.interrupt();
- // A race is possible between a task succeeding, and a subsequent timed heartbeat failing.
- // These errors can be ignored, since a task can only succeed if the synchronous taskSucceeded
- // method does not throw an exception, in which case task success is registered with the AM.
- // Leave this handling to the next getTask / actual task.
- } else {
- LOG.info("Ignoring Communication failure since task with id=" + task.getTaskAttemptID()
- + " is already complete");
- }
- }
-
- private void abortRunningTask() {
- if (!taskRunning.get()) {
- LOG.info("Task is not running");
- waitingThread.interrupt();
- return;
- }
-
- if (taskRunning.get()) {
- try {
- task.abortTask();
- } catch (Exception e) {
- LOG.warn("Error when aborting the task", e);
- try {
- sendFailure(e, "Error when aborting the task");
- } catch (Exception ignored) {
- // Ignored.
- }
- }
- }
- //Interrupt the relevant threads. TaskRunner should be interrupted preferably.
- if (isTaskRunning()) {
- LOG.info("Interrupting taskRunner=" + taskRunner.getName());
- taskRunner.interrupt();
- } else {
- LOG.info("Interrupting waitingThread=" + waitingThread.getName());
- waitingThread.interrupt();
- }
- }
-
- private boolean isTaskRunning() {
- return (taskRunning.get() && task.isRunning());
- }
-
- @Override
- public void shutdownRequested() {
- shutdownRequested.set(true);
- abortRunningTask();
- }
-
- private String getTaskDiagnosticsString(Throwable t, String message) {
- String diagnostics;
- if (t != null && message != null) {
- diagnostics = "exceptionThrown=" + ExceptionUtils.getStackTrace(t) + ", errorMessage="
- + message;
- } else if (t == null && message == null) {
- diagnostics = "Unknown error";
- } else {
- diagnostics = t != null ? "exceptionThrown=" + ExceptionUtils.getStackTrace(t)
- : " errorMessage=" + message;
- }
- return diagnostics;
- }
-
- private synchronized void maybeRegisterFirstException(Throwable t) {
- if (firstException == null) {
- firstException = t;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
deleted file mode 100644
index a99416a..0000000
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
+++ /dev/null
@@ -1,362 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.runtime.task;
-
-import static org.apache.tez.runtime.task.TaskExecutionTestHelpers.createTaskReporter;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-import org.apache.tez.dag.records.TezVertexID;
-import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
-import org.apache.tez.runtime.api.impl.InputSpec;
-import org.apache.tez.runtime.api.impl.OutputSpec;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.runtime.common.resources.ScalingAllocator;
-import org.apache.tez.runtime.task.TaskExecutionTestHelpers.TestProcessor;
-import org.apache.tez.runtime.task.TaskExecutionTestHelpers.TezTaskUmbilicalForTest;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-
-// Tests in this class cannot be run in parallel.
-public class TestTaskExecution {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestTaskExecution.class);
-
-
-
- private static final Configuration defaultConf = new Configuration();
- private static final FileSystem localFs;
- private static final Path workDir;
-
- private static final ExecutorService taskExecutor = Executors.newFixedThreadPool(1);
-
- static {
- defaultConf.set("fs.defaultFS", "file:///");
- defaultConf.set(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ALLOCATOR_CLASS,
- ScalingAllocator.class.getName());
- try {
- localFs = FileSystem.getLocal(defaultConf);
- Path wd = new Path(System.getProperty("test.build.data", "/tmp"),
- TestTaskExecution.class.getSimpleName());
- workDir = localFs.makeQualified(wd);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Before
- public void reset() {
- TestProcessor.reset();
- }
-
- @AfterClass
- public static void shutdown() {
- taskExecutor.shutdownNow();
- }
-
- @Test(timeout = 5000)
- public void testSingleSuccessfulTask() throws IOException, InterruptedException, TezException,
- ExecutionException {
- ListeningExecutorService executor = null;
- try {
- ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
- executor = MoreExecutors.listeningDecorator(rawExecutor);
- ApplicationId appId = ApplicationId.newInstance(10000, 1);
- TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
- TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
- TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
- TestProcessor.CONF_EMPTY);
- // Setup the executor
- Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
- // Signal the processor to go through
- TestProcessor.signal();
- boolean result = taskRunnerFuture.get();
- assertTrue(result);
- assertNull(taskReporter.currentCallable);
- umbilical.verifyTaskSuccessEvent();
- } finally {
- executor.shutdownNow();
- }
- }
-
- @Test(timeout = 5000)
- public void testMultipleSuccessfulTasks() throws IOException, InterruptedException, TezException,
- ExecutionException {
-
- ListeningExecutorService executor = null;
- try {
- ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
- executor = MoreExecutors.listeningDecorator(rawExecutor);
- ApplicationId appId = ApplicationId.newInstance(10000, 1);
- TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
- TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
- TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
- TestProcessor.CONF_EMPTY);
- // Setup the executor
- Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
- // Signal the processor to go through
- TestProcessor.signal();
- boolean result = taskRunnerFuture.get();
- assertTrue(result);
- assertNull(taskReporter.currentCallable);
- umbilical.verifyTaskSuccessEvent();
- umbilical.resetTrackedEvents();
-
- taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
- TestProcessor.CONF_EMPTY);
- // Setup the executor
- taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
- // Signal the processor to go through
- TestProcessor.signal();
- result = taskRunnerFuture.get();
- assertTrue(result);
- assertNull(taskReporter.currentCallable);
- umbilical.verifyTaskSuccessEvent();
- } finally {
- executor.shutdownNow();
- }
- }
-
- // test task failed due to exception in Processor
- @Test(timeout = 5000)
- public void testFailedTask() throws IOException, InterruptedException, TezException {
-
- ListeningExecutorService executor = null;
- try {
- ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
- executor = MoreExecutors.listeningDecorator(rawExecutor);
- ApplicationId appId = ApplicationId.newInstance(10000, 1);
- TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
- TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
- TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
- TestProcessor.CONF_THROW_TEZ_EXCEPTION);
- // Setup the executor
- Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
- // Signal the processor to go through
- TestProcessor.awaitStart();
- TestProcessor.signal();
- try {
- taskRunnerFuture.get();
- fail("Expecting the task to fail");
- } catch (ExecutionException e) {
- Throwable cause = e.getCause();
- LOG.info(cause.getClass().getName());
- assertTrue(cause instanceof TezException);
- }
-
- assertNull(taskReporter.currentCallable);
- umbilical.verifyTaskFailedEvent("Failure while running task:org.apache.tez.dag.api.TezException: TezException");
- } finally {
- executor.shutdownNow();
- }
- }
-
- // Test task failed due to Processor class not found
- @Test(timeout = 5000)
- public void testFailedTask2() throws IOException, InterruptedException, TezException {
-
- ListeningExecutorService executor = null;
- try {
- ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
- executor = MoreExecutors.listeningDecorator(rawExecutor);
- ApplicationId appId = ApplicationId.newInstance(10000, 1);
- TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
- TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
- TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
- "NotExitedProcessor", TestProcessor.CONF_THROW_TEZ_EXCEPTION);
- // Setup the executor
- Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
- try {
- taskRunnerFuture.get();
- } catch (ExecutionException e) {
- Throwable cause = e.getCause();
- LOG.info(cause.getClass().getName());
- assertTrue(cause instanceof TezException);
- }
- assertNull(taskReporter.currentCallable);
- umbilical.verifyTaskFailedEvent("Failure while running task:org.apache.tez.dag.api.TezUncheckedException: "
- + "Unable to load class: NotExitedProcessor");
- } finally {
- executor.shutdownNow();
- }
- }
-
- @Test(timeout = 5000)
- public void testHeartbeatException() throws IOException, InterruptedException, TezException {
-
- ListeningExecutorService executor = null;
- try {
- ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
- executor = MoreExecutors.listeningDecorator(rawExecutor);
- ApplicationId appId = ApplicationId.newInstance(10000, 1);
- TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
- TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
- TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
- TestProcessor.CONF_EMPTY);
- // Setup the executor
- Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
- // Signal the processor to go through
- TestProcessor.awaitStart();
- umbilical.signalThrowException();
- umbilical.awaitRegisteredEvent();
- // Not signaling an actual start to verify task interruption
- try {
- taskRunnerFuture.get();
- fail("Expecting the task to fail");
- } catch (ExecutionException e) {
- Throwable cause = e.getCause();
- assertTrue(cause instanceof IOException);
- assertTrue(cause.getMessage().contains(TaskExecutionTestHelpers.HEARTBEAT_EXCEPTION_STRING));
- }
- TestProcessor.awaitCompletion();
- assertTrue(TestProcessor.wasInterrupted());
- assertNull(taskReporter.currentCallable);
- // No completion events since umbilical communication already failed.
- umbilical.verifyNoCompletionEvents();
- } finally {
- executor.shutdownNow();
- }
- }
-
- @Test(timeout = 5000)
- public void testHeartbeatShouldDie() throws IOException, InterruptedException, TezException,
- ExecutionException {
-
- ListeningExecutorService executor = null;
- try {
- ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
- executor = MoreExecutors.listeningDecorator(rawExecutor);
- ApplicationId appId = ApplicationId.newInstance(10000, 1);
- TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
- TaskReporter taskReporter = createTaskReporter(appId, umbilical);
-
- TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
- TestProcessor.CONF_EMPTY);
- // Setup the executor
- Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
- // Signal the processor to go through
- TestProcessor.awaitStart();
- umbilical.signalSendShouldDie();
- umbilical.awaitRegisteredEvent();
- // Not signaling an actual start to verify task interruption
-
- boolean result = taskRunnerFuture.get();
- assertFalse(result);
-
- TestProcessor.awaitCompletion();
- assertTrue(TestProcessor.wasInterrupted());
- assertNull(taskReporter.currentCallable);
- // TODO Is this statement correct ?
- // No completion events since shouldDie was requested by the AM, which should have killed the
- // task.
- umbilical.verifyNoCompletionEvents();
- } finally {
- executor.shutdownNow();
- }
- }
-
- // Potential new tests
- // Different states - initialization failure, close failure
- // getTask states
-
- private static class TaskRunnerCallable1ForTest implements Callable<Boolean> {
- private final TezTaskRunner taskRunner;
-
- public TaskRunnerCallable1ForTest(TezTaskRunner taskRunner) {
- this.taskRunner = taskRunner;
- }
-
- @Override
- public Boolean call() throws Exception {
- return taskRunner.run();
- }
- }
-
-
-
-
-
- private TezTaskRunner createTaskRunner(ApplicationId appId, TezTaskUmbilicalForTest umbilical,
- TaskReporter taskReporter, ListeningExecutorService executor, byte[] processorConf)
- throws IOException {
- return createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.class.getName(),
- processorConf);
- }
-
- private TezTaskRunner createTaskRunner(ApplicationId appId, TezTaskUmbilicalForTest umbilical,
- TaskReporter taskReporter, ListeningExecutorService executor, String processorClass, byte[] processorConf) throws IOException{
- TezConfiguration tezConf = new TezConfiguration(defaultConf);
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- Path testDir = new Path(workDir, UUID.randomUUID().toString());
- String[] localDirs = new String[] { testDir.toString() };
-
- TezDAGID dagId = TezDAGID.getInstance(appId, 1);
- TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
- TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
- TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
- ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass)
- .setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
- TaskSpec taskSpec = new TaskSpec(taskAttemptId, "dagName", "vertexName", -1, processorDescriptor,
- new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null);
-
- TezTaskRunner taskRunner = new TezTaskRunner(tezConf, ugi, localDirs, taskSpec, 1,
- new HashMap<String, ByteBuffer>(), new HashMap<String, String>(), HashMultimap.<String, String> create(), taskReporter,
- executor, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory());
- return taskRunner;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
index 12d9d3f..ce9095a 100644
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
@@ -89,7 +89,7 @@ public class TestTaskExecution2 {
try {
localFs = FileSystem.getLocal(defaultConf);
Path wd = new Path(System.getProperty("test.build.data", "/tmp"),
- TestTaskExecution.class.getSimpleName());
+ TestTaskExecution2.class.getSimpleName());
workDir = localFs.makeQualified(wd);
} catch (IOException e) {
throw new RuntimeException(e);
[2/7] tez git commit: TEZ-2708. Rename classes and variables post
TEZ-2003 changes. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
index 0a02f9e..0e90681 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
@@ -68,7 +68,7 @@ import org.apache.tez.dag.app.ClusterInfo;
import org.apache.tez.dag.app.ContainerContext;
import org.apache.tez.dag.app.ContainerHeartbeatHandler;
import org.apache.tez.dag.app.DAGAppMasterState;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.app.rm.YarnTaskSchedulerService.CookieContainerRequest;
import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.AMRMClientAsyncForTest;
@@ -76,7 +76,7 @@ import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.AMRMClientForTest;
import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.AlwaysMatchesContainerMatcher;
import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.CapturingEventHandler;
import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.TaskSchedulerContextDrainable;
-import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.TaskSchedulerEventHandlerForTest;
+import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.TaskSchedulerManagerForTest;
import org.apache.tez.dag.app.rm.TestTaskSchedulerHelpers.TaskSchedulerWithDrainableContext;
import org.apache.tez.dag.app.rm.container.AMContainerEventAssignTA;
import org.apache.tez.dag.app.rm.container.AMContainerEventStopRequest;
@@ -136,7 +136,7 @@ public class TestContainerReuse {
doReturn(conf).when(appContext).getAMConf();
AMContainerMap amContainerMap = new AMContainerMap(
mock(ContainerHeartbeatHandler.class),
- mock(TaskAttemptListener.class),
+ mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appContext);
AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
doReturn(amContainerMap).when(appContext).getAllContainers();
@@ -145,18 +145,18 @@ public class TestContainerReuse {
doReturn(dagID).when(appContext).getCurrentDAGID();
doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
- TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
- new TaskSchedulerEventHandlerForTest(
+ TaskSchedulerManager taskSchedulerManagerReal =
+ new TaskSchedulerManagerForTest(
appContext, eventHandler, rmClient,
new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(conf));
- TaskSchedulerEventHandler taskSchedulerEventHandler =
- spy(taskSchedulerEventHandlerReal);
- taskSchedulerEventHandler.init(conf);
- taskSchedulerEventHandler.start();
+ TaskSchedulerManager taskSchedulerManager =
+ spy(taskSchedulerManagerReal);
+ taskSchedulerManager.init(conf);
+ taskSchedulerManager.start();
TaskSchedulerWithDrainableContext taskScheduler =
(TaskSchedulerWithDrainableContext)
- ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+ ((TaskSchedulerManagerForTest) taskSchedulerManager)
.getSpyTaskScheduler();
TaskSchedulerContextDrainable drainableAppCallback =
taskScheduler.getDrainableAppCallback();
@@ -192,10 +192,10 @@ public class TestContainerReuse {
defaultRack, priority);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrTa11);
+ taskSchedulerManager.handleEvent(lrTa11);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrTa21);
+ taskSchedulerManager.handleEvent(lrTa21);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
Container containerHost1 = createContainer(1, host1[0], resource, priority);
@@ -206,28 +206,28 @@ public class TestContainerReuse {
Lists.newArrayList(containerHost1, containerHost2));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler).taskAllocated(
+ verify(taskSchedulerManager).taskAllocated(
eq(0), eq(ta11), any(Object.class), eq(containerHost1));
- verify(taskSchedulerEventHandler).taskAllocated(
+ verify(taskSchedulerManager).taskAllocated(
eq(0), eq(ta21), any(Object.class), eq(containerHost2));
// Adding the event later so that task1 assigned to containerHost1
// is deterministic.
- taskSchedulerEventHandler.handleEvent(lrTa31);
+ taskSchedulerManager.handleEvent(lrTa31);
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(
ta11, containerHost1.getId(), TaskAttemptState.SUCCEEDED, null, null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta11, true, null, null);
- verify(taskSchedulerEventHandler, times(1)).taskAllocated(
+ verify(taskSchedulerManager, times(1)).taskAllocated(
eq(0), eq(ta31), any(Object.class), eq(containerHost1));
verify(rmClient, times(0)).releaseAssignedContainer(
eq(containerHost1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
eventHandler.reset();
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta21, containerHost2.getId(),
TaskAttemptState.SUCCEEDED, null, null, 0));
@@ -235,7 +235,7 @@ public class TestContainerReuse {
Throwable exception = null;
while (System.currentTimeMillis() < currentTs + 5000l) {
try {
- verify(taskSchedulerEventHandler,
+ verify(taskSchedulerManager,
times(1)).containerBeingReleased(eq(0), eq(containerHost2.getId()));
exception = null;
break;
@@ -245,7 +245,7 @@ public class TestContainerReuse {
}
assertTrue("containerHost2 was not released", exception == null);
taskScheduler.shutdown();
- taskSchedulerEventHandler.close();
+ taskSchedulerManager.close();
}
@Test(timeout = 15000l)
@@ -272,7 +272,7 @@ public class TestContainerReuse {
doReturn(new Configuration(false)).when(appContext).getAMConf();
AMContainerMap amContainerMap = new AMContainerMap(
mock(ContainerHeartbeatHandler.class),
- mock(TaskAttemptListener.class),
+ mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appContext);
AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
doReturn(amContainerMap).when(appContext).getAllContainers();
@@ -281,17 +281,17 @@ public class TestContainerReuse {
doReturn(dagID).when(appContext).getCurrentDAGID();
doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
- TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
- new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient,
+ TaskSchedulerManager taskSchedulerManagerReal =
+ new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient,
new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(conf));
- TaskSchedulerEventHandler taskSchedulerEventHandler =
- spy(taskSchedulerEventHandlerReal);
- taskSchedulerEventHandler.init(conf);
- taskSchedulerEventHandler.start();
+ TaskSchedulerManager taskSchedulerManager =
+ spy(taskSchedulerManagerReal);
+ taskSchedulerManager.init(conf);
+ taskSchedulerManager.start();
TaskSchedulerWithDrainableContext taskScheduler =
(TaskSchedulerWithDrainableContext)
- ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+ ((TaskSchedulerManagerForTest) taskSchedulerManager)
.getSpyTaskScheduler();
TaskSchedulerContextDrainable drainableAppCallback =
taskScheduler.getDrainableAppCallback();
@@ -321,10 +321,10 @@ public class TestContainerReuse {
taID31, ta31, resource, host1, defaultRack, priority);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrTa11);
+ taskSchedulerManager.handleEvent(lrTa11);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrTa21);
+ taskSchedulerManager.handleEvent(lrTa21);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
Container containerHost1 = createContainer(1, host1[0], resource, priority);
@@ -334,26 +334,26 @@ public class TestContainerReuse {
taskScheduler.onContainersAllocated(Lists.newArrayList(containerHost1, containerHost2));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta11), any(Object.class), eq(containerHost1));
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta21), any(Object.class),
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta11), any(Object.class), eq(containerHost1));
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta21), any(Object.class),
eq(containerHost2));
// Adding the event later so that task1 assigned to containerHost1 is deterministic.
- taskSchedulerEventHandler.handleEvent(lrTa31);
+ taskSchedulerManager.handleEvent(lrTa31);
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta21, containerHost2.getId(),
TaskAttemptState.SUCCEEDED, null, null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta21, true, null, null);
- verify(taskSchedulerEventHandler, times(0)).taskAllocated(
+ verify(taskSchedulerManager, times(0)).taskAllocated(
eq(0), eq(ta31), any(Object.class), eq(containerHost2));
verify(rmClient, times(1)).releaseAssignedContainer(
eq(containerHost2.getId()));
eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
taskScheduler.shutdown();
- taskSchedulerEventHandler.close();
+ taskSchedulerManager.close();
}
@Test(timeout = 10000l)
@@ -375,7 +375,7 @@ public class TestContainerReuse {
AppContext appContext = mock(AppContext.class);
doReturn(new Configuration(false)).when(appContext).getAMConf();
AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class),
- mock(TaskAttemptListener.class), new ContainerContextMatcher(), appContext);
+ mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
doReturn(amContainerMap).when(appContext).getAllContainers();
doReturn(amNodeTracker).when(appContext).getNodeTracker();
@@ -383,12 +383,13 @@ public class TestContainerReuse {
doReturn(dagID).when(appContext).getCurrentDAGID();
doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
- TaskSchedulerEventHandler taskSchedulerEventHandlerReal = new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
- TaskSchedulerEventHandler taskSchedulerEventHandler = spy(taskSchedulerEventHandlerReal);
- taskSchedulerEventHandler.init(tezConf);
- taskSchedulerEventHandler.start();
+ TaskSchedulerManager
+ taskSchedulerManagerReal = new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
+ TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal);
+ taskSchedulerManager.init(tezConf);
+ taskSchedulerManager.start();
- TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+ TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerManagerForTest) taskSchedulerManager)
.getSpyTaskScheduler();
TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
AtomicBoolean drainNotifier = new AtomicBoolean(false);
@@ -423,10 +424,10 @@ public class TestContainerReuse {
TaskAttempt ta14 = mock(TaskAttempt.class);
AMSchedulerEventTALaunchRequest lrEvent4 = createLaunchRequestEvent(taID14, ta14, resource1, host2, racks, priority1);
- taskSchedulerEventHandler.handleEvent(lrEvent1);
- taskSchedulerEventHandler.handleEvent(lrEvent2);
- taskSchedulerEventHandler.handleEvent(lrEvent3);
- taskSchedulerEventHandler.handleEvent(lrEvent4);
+ taskSchedulerManager.handleEvent(lrEvent1);
+ taskSchedulerManager.handleEvent(lrEvent2);
+ taskSchedulerManager.handleEvent(lrEvent3);
+ taskSchedulerManager.handleEvent(lrEvent4);
Container container1 = createContainer(1, "host1", resource1, priority1);
@@ -435,39 +436,39 @@ public class TestContainerReuse {
taskScheduler.onContainersAllocated(Collections.singletonList(container1));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta11), any(Object.class),
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta11), any(Object.class),
eq(container1));
// Task assigned to container completed successfully. Container should be re-used.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED, null,
null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta11, true, null, null);
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta12), any(Object.class), eq(container1));
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta12), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
eventHandler.reset();
// Task assigned to container completed successfully.
// Verify reuse across hosts.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta12, container1.getId(), TaskAttemptState.SUCCEEDED, null,
null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta12, true, null, null);
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta13), any(Object.class),
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta13), any(Object.class),
eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
eventHandler.reset();
// Verify no re-use if a previous task fails.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta13, container1.getId(), TaskAttemptState.FAILED, null,
"TIMEOUT", 0));
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(0), eq(ta14), any(Object.class),
+ verify(taskSchedulerManager, times(0)).taskAllocated(eq(0), eq(ta14), any(Object.class),
eq(container1));
verifyDeAllocateTask(taskScheduler, ta13, false, null, "TIMEOUT");
verify(rmClient).releaseAssignedContainer(eq(container1.getId()));
@@ -481,11 +482,11 @@ public class TestContainerReuse {
taskScheduler.onContainersAllocated(Collections.singletonList(container2));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta14), any(Object.class),
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta14), any(Object.class),
eq(container2));
// Task assigned to container completed successfully. No pending requests. Container should be released.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta14, container2.getId(), TaskAttemptState.SUCCEEDED, null,
null, 0));
drainableAppCallback.drain();
@@ -495,7 +496,7 @@ public class TestContainerReuse {
eventHandler.reset();
taskScheduler.shutdown();
- taskSchedulerEventHandler.close();
+ taskSchedulerManager.close();
}
@Test(timeout = 10000l)
@@ -521,7 +522,7 @@ public class TestContainerReuse {
AppContext appContext = mock(AppContext.class);
doReturn(new Configuration(false)).when(appContext).getAMConf();
AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class),
- mock(TaskAttemptListener.class), new ContainerContextMatcher(), appContext);
+ mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
doReturn(amContainerMap).when(appContext).getAllContainers();
doReturn(amNodeTracker).when(appContext).getNodeTracker();
@@ -530,14 +531,14 @@ public class TestContainerReuse {
doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
//Use ContainerContextMatcher here. Otherwise it would not match the JVM options
- TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
- new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient, new ContainerContextMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
- TaskSchedulerEventHandler taskSchedulerEventHandler = spy(taskSchedulerEventHandlerReal);
- taskSchedulerEventHandler.init(tezConf);
- taskSchedulerEventHandler.start();
+ TaskSchedulerManager taskSchedulerManagerReal =
+ new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, new ContainerContextMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
+ TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal);
+ taskSchedulerManager.init(tezConf);
+ taskSchedulerManager.start();
TaskSchedulerWithDrainableContext taskScheduler =
- (TaskSchedulerWithDrainableContext) ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+ (TaskSchedulerWithDrainableContext) ((TaskSchedulerManagerForTest) taskSchedulerManager)
.getSpyTaskScheduler();
TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
AtomicBoolean drainNotifier = new AtomicBoolean(false);
@@ -573,10 +574,10 @@ public class TestContainerReuse {
createLaunchRequestEvent(taID12, ta12, resource1, host1, racks, priority1);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrEvent1);
+ taskSchedulerManager.handleEvent(lrEvent1);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrEvent2);
+ taskSchedulerManager.handleEvent(lrEvent2);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
Container container1 = createContainer(1, "host1", resource1, priority1);
@@ -586,16 +587,16 @@ public class TestContainerReuse {
taskScheduler.onContainersAllocated(Collections.singletonList(container1));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta11), any(Object.class),
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta11), any(Object.class),
eq(container1));
// First task had profiling on. This container can not be reused further.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED, null,
null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta11, true, null, null);
- verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(0), eq(ta12), any(Object.class),
+ verify(taskSchedulerManager, times(0)).taskAllocated(eq(0), eq(ta12), any(Object.class),
eq(container1));
verify(rmClient, times(1)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
@@ -623,10 +624,10 @@ public class TestContainerReuse {
Container container2 = createContainer(2, "host2", resource1, priority1);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrEvent3);
+ taskSchedulerManager.handleEvent(lrEvent3);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrEvent4);
+ taskSchedulerManager.handleEvent(lrEvent4);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
// Container started
@@ -634,16 +635,16 @@ public class TestContainerReuse {
taskScheduler.onContainersAllocated(Collections.singletonList(container2));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta13), any(Object.class), eq(container2));
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta13), any(Object.class), eq(container2));
// Verify that the container can not be reused when profiling option is turned on
// Even for 2 tasks having same profiling option can have container reusability.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta13, container2.getId(), TaskAttemptState.SUCCEEDED, null,
null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta13, true, null, null);
- verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(0), eq(ta14), any(Object.class),
+ verify(taskSchedulerManager, times(0)).taskAllocated(eq(0), eq(ta14), any(Object.class),
eq(container2));
verify(rmClient, times(1)).releaseAssignedContainer(eq(container2.getId()));
eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
@@ -675,27 +676,27 @@ public class TestContainerReuse {
// Container started
Container container3 = createContainer(2, "host3", resource1, priority1);
- taskSchedulerEventHandler.handleEvent(lrEvent5);
- taskSchedulerEventHandler.handleEvent(lrEvent6);
+ taskSchedulerManager.handleEvent(lrEvent5);
+ taskSchedulerManager.handleEvent(lrEvent6);
drainNotifier.set(false);
taskScheduler.onContainersAllocated(Collections.singletonList(container3));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta15), any(Object.class),
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta15), any(Object.class),
eq(container3));
//Ensure task 6 (of vertex 1) is allocated to same container
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta15, container3.getId(), TaskAttemptState.SUCCEEDED, null,
null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta15, true, null, null);
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta16), any(Object.class), eq(container3));
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta16), any(Object.class), eq(container3));
eventHandler.reset();
taskScheduler.shutdown();
- taskSchedulerEventHandler.close();
+ taskSchedulerManager.close();
}
@Test(timeout = 30000l)
@@ -721,7 +722,7 @@ public class TestContainerReuse {
doReturn(new Configuration(false)).when(appContext).getAMConf();
AMContainerMap amContainerMap = new AMContainerMap(
mock(ContainerHeartbeatHandler.class),
- mock(TaskAttemptListener.class),
+ mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appContext);
AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
doReturn(amContainerMap).when(appContext).getAllContainers();
@@ -730,18 +731,18 @@ public class TestContainerReuse {
doReturn(dagID).when(appContext).getCurrentDAGID();
doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
- TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
- new TaskSchedulerEventHandlerForTest(
+ TaskSchedulerManager taskSchedulerManagerReal =
+ new TaskSchedulerManagerForTest(
appContext, eventHandler, rmClient,
new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
- TaskSchedulerEventHandler taskSchedulerEventHandler =
- spy(taskSchedulerEventHandlerReal);
- taskSchedulerEventHandler.init(tezConf);
- taskSchedulerEventHandler.start();
+ TaskSchedulerManager taskSchedulerManager =
+ spy(taskSchedulerManagerReal);
+ taskSchedulerManager.init(tezConf);
+ taskSchedulerManager.start();
TaskSchedulerWithDrainableContext taskScheduler =
(TaskSchedulerWithDrainableContext)
- ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+ ((TaskSchedulerManagerForTest) taskSchedulerManager)
.getSpyTaskScheduler();
TaskSchedulerContextDrainable drainableAppCallback =
taskScheduler.getDrainableAppCallback();
@@ -772,7 +773,7 @@ public class TestContainerReuse {
// Send launch request for task 1 only, deterministic assignment to this task.
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrEvent11);
+ taskSchedulerManager.handleEvent(lrEvent11);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
Container container1 = createContainer(1, "randomHost", resource1, priority);
@@ -782,21 +783,21 @@ public class TestContainerReuse {
taskScheduler.onContainersAllocated(Collections.singletonList(container1));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler).taskAllocated(
+ verify(taskSchedulerManager).taskAllocated(
eq(0), eq(ta11), any(Object.class), eq(container1));
// Send launch request for task2 (vertex2)
- taskSchedulerEventHandler.handleEvent(lrEvent12);
+ taskSchedulerManager.handleEvent(lrEvent12);
// Task assigned to container completed successfully.
// Container should not be immediately assigned to task 2
// until delay expires.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta11, container1.getId(),
TaskAttemptState.SUCCEEDED, null, null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta11, true, null, null);
- verify(taskSchedulerEventHandler, times(0)).taskAllocated(
+ verify(taskSchedulerManager, times(0)).taskAllocated(
eq(0), eq(ta12), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
@@ -804,11 +805,11 @@ public class TestContainerReuse {
LOG.info("Sleeping to ensure that the scheduling loop runs");
Thread.sleep(3000l);
- verify(taskSchedulerEventHandler).taskAllocated(
+ verify(taskSchedulerManager).taskAllocated(
eq(0), eq(ta12), any(Object.class), eq(container1));
// TA12 completed.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta12, container1.getId(),
TaskAttemptState.SUCCEEDED, null, null, 0));
drainableAppCallback.drain();
@@ -818,7 +819,7 @@ public class TestContainerReuse {
eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
taskScheduler.shutdown();
- taskSchedulerEventHandler.close();
+ taskSchedulerManager.close();
}
@Test(timeout = 30000l)
@@ -845,7 +846,7 @@ public class TestContainerReuse {
doReturn(new Configuration(false)).when(appContext).getAMConf();
AMContainerMap amContainerMap = new AMContainerMap(
mock(ContainerHeartbeatHandler.class),
- mock(TaskAttemptListener.class),
+ mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appContext);
AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
doReturn(amContainerMap).when(appContext).getAllContainers();
@@ -855,17 +856,17 @@ public class TestContainerReuse {
doReturn(dagID).when(appContext).getCurrentDAGID();
doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
- TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
- new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient,
+ TaskSchedulerManager taskSchedulerManagerReal =
+ new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient,
new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
- TaskSchedulerEventHandler taskSchedulerEventHandler =
- spy(taskSchedulerEventHandlerReal);
- taskSchedulerEventHandler.init(tezConf);
- taskSchedulerEventHandler.start();
+ TaskSchedulerManager taskSchedulerManager =
+ spy(taskSchedulerManagerReal);
+ taskSchedulerManager.init(tezConf);
+ taskSchedulerManager.start();
TaskSchedulerWithDrainableContext taskScheduler =
(TaskSchedulerWithDrainableContext)
- ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+ ((TaskSchedulerManagerForTest) taskSchedulerManager)
.getSpyTaskScheduler();
TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
@@ -900,7 +901,7 @@ public class TestContainerReuse {
// Send launch request for task 1 onle, deterministic assignment to this task.
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrEvent11);
+ taskSchedulerManager.handleEvent(lrEvent11);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
Container container1 = createContainer(1, host1[0], resource1, priority1);
@@ -910,25 +911,25 @@ public class TestContainerReuse {
taskScheduler.onContainersAllocated(Collections.singletonList(container1));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler).taskAllocated(
+ verify(taskSchedulerManager).taskAllocated(
eq(0), eq(ta11), any(Object.class), eq(container1));
// Send launch request for task2 (vertex2)
- taskSchedulerEventHandler.handleEvent(lrEvent21);
+ taskSchedulerManager.handleEvent(lrEvent21);
// Task assigned to container completed successfully.
// Container should be assigned to task21.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta11, container1.getId(),
TaskAttemptState.SUCCEEDED, null, null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta11, true, null, null);
- verify(taskSchedulerEventHandler).taskAllocated(
+ verify(taskSchedulerManager).taskAllocated(
eq(0), eq(ta21), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
// Task 2 completes.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta21, container1.getId(),
TaskAttemptState.SUCCEEDED, null, null, 0));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
@@ -939,7 +940,7 @@ public class TestContainerReuse {
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
taskScheduler.shutdown();
- taskSchedulerEventHandler.close();
+ taskSchedulerManager.close();
}
@Test(timeout = 30000l)
@@ -962,7 +963,7 @@ public class TestContainerReuse {
doReturn(new Configuration(false)).when(appContext).getAMConf();
ChangingDAGIDAnswer dagIDAnswer = new ChangingDAGIDAnswer(dagID1);
AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class),
- mock(TaskAttemptListener.class), new ContainerContextMatcher(), appContext);
+ mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
doReturn(amContainerMap).when(appContext).getAllContainers();
doReturn(amNodeTracker).when(appContext).getNodeTracker();
@@ -971,12 +972,13 @@ public class TestContainerReuse {
doAnswer(dagIDAnswer).when(appContext).getCurrentDAGID();
doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
- TaskSchedulerEventHandler taskSchedulerEventHandlerReal = new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
- TaskSchedulerEventHandler taskSchedulerEventHandler = spy(taskSchedulerEventHandlerReal);
- taskSchedulerEventHandler.init(tezConf);
- taskSchedulerEventHandler.start();
+ TaskSchedulerManager
+ taskSchedulerManagerReal = new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
+ TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal);
+ taskSchedulerManager.init(tezConf);
+ taskSchedulerManager.start();
- TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+ TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerManagerForTest) taskSchedulerManager)
.getSpyTaskScheduler();
TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
AtomicBoolean drainNotifier = new AtomicBoolean(false);
@@ -1013,10 +1015,10 @@ public class TestContainerReuse {
AMSchedulerEventTALaunchRequest lrEvent12 = createLaunchRequestEvent(taID112, ta112, resource1, host1, racks, priority1, dag1LRs);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrEvent11);
+ taskSchedulerManager.handleEvent(lrEvent11);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrEvent12);
+ taskSchedulerManager.handleEvent(lrEvent12);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
Container container1 = createContainer(1, "host1", resource1, priority1);
@@ -1026,17 +1028,17 @@ public class TestContainerReuse {
taskScheduler.onContainersAllocated(Collections.singletonList(container1));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta111), any(Object.class), eq(container1));
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta111), any(Object.class), eq(container1));
assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
assertEquals(1, assignEvent.getRemoteTaskLocalResources().size());
// Task assigned to container completed successfully. Container should be re-used.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED, null,
null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta111, true, null, null);
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta112), any(Object.class), eq(container1));
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta112), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
@@ -1045,7 +1047,7 @@ public class TestContainerReuse {
// Task assigned to container completed successfully.
// Verify reuse across hosts.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED, null,
null, 0));
drainableAppCallback.drain();
@@ -1074,14 +1076,14 @@ public class TestContainerReuse {
TaskAttempt ta212 = mock(TaskAttempt.class);
AMSchedulerEventTALaunchRequest lrEvent22 = createLaunchRequestEvent(taID212, ta212, resource1, host1, racks, priority1, dag2LRs);
- taskSchedulerEventHandler.handleEvent(lrEvent21);
- taskSchedulerEventHandler.handleEvent(lrEvent22);
+ taskSchedulerManager.handleEvent(lrEvent21);
+ taskSchedulerManager.handleEvent(lrEvent22);
drainableAppCallback.drain();
// TODO This is terrible, need a better way to ensure the scheduling loop has run
LOG.info("Sleeping to ensure that the scheduling loop runs");
Thread.sleep(6000l);
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta211), any(Object.class),
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta211), any(Object.class),
eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
@@ -1089,12 +1091,12 @@ public class TestContainerReuse {
assertEquals(2, assignEvent.getRemoteTaskLocalResources().size());
eventHandler.reset();
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta211, container1.getId(), TaskAttemptState.SUCCEEDED, null,
null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta211, true, null, null);
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta212), any(Object.class), eq(container1));
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta212), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
@@ -1102,7 +1104,7 @@ public class TestContainerReuse {
eventHandler.reset();
taskScheduler.shutdown();
- taskSchedulerEventHandler.close();
+ taskSchedulerManager.close();
}
@Test(timeout = 30000l)
@@ -1125,7 +1127,7 @@ public class TestContainerReuse {
doReturn(new Configuration(false)).when(appContext).getAMConf();
ChangingDAGIDAnswer dagIDAnswer = new ChangingDAGIDAnswer(dagID1);
AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class),
- mock(TaskAttemptListener.class), new ContainerContextMatcher(), appContext);
+ mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
doReturn(amContainerMap).when(appContext).getAllContainers();
doReturn(amNodeTracker).when(appContext).getNodeTracker();
@@ -1134,14 +1136,14 @@ public class TestContainerReuse {
doAnswer(dagIDAnswer).when(appContext).getCurrentDAGID();
doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
- TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
- new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient,
+ TaskSchedulerManager taskSchedulerManagerReal =
+ new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient,
new ContainerContextMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
- TaskSchedulerEventHandler taskSchedulerEventHandler = spy(taskSchedulerEventHandlerReal);
- taskSchedulerEventHandler.init(tezConf);
- taskSchedulerEventHandler.start();
+ TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal);
+ taskSchedulerManager.init(tezConf);
+ taskSchedulerManager.start();
- TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+ TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerManagerForTest) taskSchedulerManager)
.getSpyTaskScheduler();
TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
AtomicBoolean drainNotifier = new AtomicBoolean(false);
@@ -1185,10 +1187,10 @@ public class TestContainerReuse {
AMSchedulerEventTALaunchRequest lrEvent12 = createLaunchRequestEvent(taID112, ta112, resource1, host1, racks, priority1, v12LR);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrEvent11);
+ taskSchedulerManager.handleEvent(lrEvent11);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainNotifier.set(false);
- taskSchedulerEventHandler.handleEvent(lrEvent12);
+ taskSchedulerManager.handleEvent(lrEvent12);
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
Container container1 = createContainer(1, "host1", resource1, priority1);
@@ -1199,17 +1201,17 @@ public class TestContainerReuse {
taskScheduler.onContainersAllocated(Collections.singletonList(container1));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta111), any(Object.class), eq(container1));
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta111), any(Object.class), eq(container1));
assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
assertEquals(1, assignEvent.getRemoteTaskLocalResources().size());
// Task assigned to container completed successfully. Container should be re-used.
- taskSchedulerEventHandler.handleEvent(
+ taskSchedulerManager.handleEvent(
new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED, null,
null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta111, true, null, null);
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta112), any(Object.class), eq(container1));
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta112), any(Object.class), eq(container1));
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
@@ -1218,7 +1220,7 @@ public class TestContainerReuse {
// Task assigned to container completed successfully.
// Verify reuse across hosts.
- taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED, null, null, 0));
+ taskSchedulerManager.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED, null, null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta112, true, null, null);
verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
@@ -1243,7 +1245,7 @@ public class TestContainerReuse {
AMSchedulerEventTALaunchRequest lrEvent21 = createLaunchRequestEvent(taID211, ta211, resource1,
host1, racks, priority1, v21LR);
- taskSchedulerEventHandler.handleEvent(lrEvent21);
+ taskSchedulerManager.handleEvent(lrEvent21);
drainableAppCallback.drain();
// TODO This is terrible, need a better way to ensure the scheduling loop has run
@@ -1254,11 +1256,11 @@ public class TestContainerReuse {
Thread.sleep(6000l);
verify(rmClient, times(1)).releaseAssignedContainer(eq(container1.getId()));
- verify(taskSchedulerEventHandler).taskAllocated(eq(0), eq(ta211), any(Object.class), eq(container2));
+ verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta211), any(Object.class), eq(container2));
eventHandler.reset();
taskScheduler.shutdown();
- taskSchedulerEventHandler.close();
+ taskSchedulerManager.close();
}
@Test(timeout = 10000l)
@@ -1281,7 +1283,7 @@ public class TestContainerReuse {
AppContext appContext = mock(AppContext.class);
doReturn(new Configuration(false)).when(appContext).getAMConf();
AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class),
- mock(TaskAttemptListener.class), new ContainerContextMatcher(), appContext);
+ mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
doReturn(amContainerMap).when(appContext).getAllContainers();
doReturn(amNodeTracker).when(appContext).getNodeTracker();
@@ -1290,15 +1292,15 @@ public class TestContainerReuse {
doReturn(dagID).when(appContext).getCurrentDAGID();
doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
- TaskSchedulerEventHandler taskSchedulerEventHandlerReal =
- new TaskSchedulerEventHandlerForTest(appContext, eventHandler, rmClient,
+ TaskSchedulerManager taskSchedulerManagerReal =
+ new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient,
new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
- TaskSchedulerEventHandler taskSchedulerEventHandler = spy(taskSchedulerEventHandlerReal);
- taskSchedulerEventHandler.init(tezConf);
- taskSchedulerEventHandler.start();
+ TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal);
+ taskSchedulerManager.init(tezConf);
+ taskSchedulerManager.start();
TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext)
- ((TaskSchedulerEventHandlerForTest) taskSchedulerEventHandler)
+ ((TaskSchedulerManagerForTest) taskSchedulerManager)
.getSpyTaskScheduler();
TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
AtomicBoolean drainNotifier = new AtomicBoolean(false);
@@ -1318,7 +1320,7 @@ public class TestContainerReuse {
TaskAttempt ta11 = mock(TaskAttempt.class);
AMSchedulerEventTALaunchRequest lrEvent1 = createLaunchRequestEvent(taID11, ta11, resource1,
host1, racks, priority1);
- taskSchedulerEventHandler.handleEvent(lrEvent1);
+ taskSchedulerManager.handleEvent(lrEvent1);
Container container1 = createContainer(1, "host1", resource1, priority1);
@@ -1326,10 +1328,10 @@ public class TestContainerReuse {
drainNotifier.set(false);
taskScheduler.onContainersAllocated(Collections.singletonList(container1));
drainableAppCallback.drain();
- verify(taskSchedulerEventHandler, times(0)).taskAllocated(eq(0), eq(ta11),
+ verify(taskSchedulerManager, times(0)).taskAllocated(eq(0), eq(ta11),
any(Object.class), eq(container1));
taskScheduler.shutdown();
- taskSchedulerEventHandler.close();
+ taskSchedulerManager.close();
}
private Container createContainer(int id, String host, Resource resource, Priority priority) {
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
deleted file mode 100644
index c85be6c..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
+++ /dev/null
@@ -1,707 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.common.ContainerSignatureMatcher;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TaskLocationHint;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.client.DAGClientServer;
-import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
-import org.apache.tez.dag.app.dag.TaskAttempt;
-import org.apache.tez.dag.app.dag.impl.TaskAttemptImpl;
-import org.apache.tez.dag.app.dag.impl.TaskImpl;
-import org.apache.tez.dag.app.dag.impl.VertexImpl;
-import org.apache.tez.dag.app.rm.container.AMContainer;
-import org.apache.tez.dag.app.rm.container.AMContainerEventAssignTA;
-import org.apache.tez.dag.app.rm.container.AMContainerEventCompleted;
-import org.apache.tez.dag.app.rm.container.AMContainerEventType;
-import org.apache.tez.dag.app.rm.container.AMContainerMap;
-import org.apache.tez.dag.app.rm.container.AMContainerState;
-import org.apache.tez.dag.app.web.WebUIService;
-import org.apache.tez.dag.records.TaskAttemptTerminationCause;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-import org.apache.tez.dag.records.TezVertexID;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.serviceplugins.api.TaskScheduler;
-import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-@SuppressWarnings("rawtypes")
-public class TestTaskSchedulerEventHandler {
-
- class TestEventHandler implements EventHandler{
- List<Event> events = Lists.newLinkedList();
- @Override
- public void handle(Event event) {
- events.add(event);
- }
- }
-
- class MockTaskSchedulerEventHandler extends TaskSchedulerEventHandler {
-
- final AtomicBoolean notify = new AtomicBoolean(false);
-
- public MockTaskSchedulerEventHandler(AppContext appContext,
- DAGClientServer clientService, EventHandler eventHandler,
- ContainerSignatureMatcher containerSignatureMatcher, WebUIService webUI) {
- super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI,
- Lists.newArrayList(new NamedEntityDescriptor("FakeDescriptor", null)), false);
- }
-
- @Override
- protected void instantiateSchedulers(String host, int port, String trackingUrl,
- AppContext appContext) {
- taskSchedulers[0] = mockTaskScheduler;
- taskSchedulerServiceWrappers[0] = new ServicePluginLifecycleAbstractService<>(taskSchedulers[0]);
- }
-
- @Override
- protected void notifyForTest() {
- synchronized (notify) {
- notify.set(true);
- notify.notifyAll();
- }
- }
-
- }
-
- AppContext mockAppContext;
- DAGClientServer mockClientService;
- TestEventHandler mockEventHandler;
- ContainerSignatureMatcher mockSigMatcher;
- MockTaskSchedulerEventHandler schedulerHandler;
- TaskScheduler mockTaskScheduler;
- AMContainerMap mockAMContainerMap;
- WebUIService mockWebUIService;
-
- @Before
- public void setup() {
- mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
- doReturn(new Configuration(false)).when(mockAppContext).getAMConf();
- mockClientService = mock(DAGClientServer.class);
- mockEventHandler = new TestEventHandler();
- mockSigMatcher = mock(ContainerSignatureMatcher.class);
- mockTaskScheduler = mock(TaskScheduler.class);
- mockAMContainerMap = mock(AMContainerMap.class);
- mockWebUIService = mock(WebUIService.class);
- when(mockAppContext.getAllContainers()).thenReturn(mockAMContainerMap);
- when(mockClientService.getBindAddress()).thenReturn(new InetSocketAddress(10000));
- schedulerHandler = new MockTaskSchedulerEventHandler(
- mockAppContext, mockClientService, mockEventHandler, mockSigMatcher, mockWebUIService);
- }
-
- @Test(timeout = 5000)
- public void testSimpleAllocate() throws Exception {
- Configuration conf = new Configuration(false);
- schedulerHandler.init(conf);
- schedulerHandler.start();
-
- TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
- TezTaskAttemptID mockAttemptId = mock(TezTaskAttemptID.class);
- when(mockAttemptId.getId()).thenReturn(0);
- when(mockTaskAttempt.getID()).thenReturn(mockAttemptId);
- Resource resource = Resource.newInstance(1024, 1);
- ContainerContext containerContext =
- new ContainerContext(new HashMap<String, LocalResource>(), new Credentials(),
- new HashMap<String, String>(), "");
- int priority = 10;
- TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(), null);
-
- ContainerId mockCId = mock(ContainerId.class);
- Container container = mock(Container.class);
- when(container.getId()).thenReturn(mockCId);
-
- AMContainer mockAMContainer = mock(AMContainer.class);
- when(mockAMContainer.getContainerId()).thenReturn(mockCId);
- when(mockAMContainer.getState()).thenReturn(AMContainerState.IDLE);
-
- when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
-
- AMSchedulerEventTALaunchRequest lr =
- new AMSchedulerEventTALaunchRequest(mockAttemptId, resource, null, mockTaskAttempt, locHint,
- priority, containerContext, 0, 0, 0);
- schedulerHandler.taskAllocated(0, mockTaskAttempt, lr, container);
- assertEquals(2, mockEventHandler.events.size());
- assertTrue(mockEventHandler.events.get(1) instanceof AMContainerEventAssignTA);
- AMContainerEventAssignTA assignEvent =
- (AMContainerEventAssignTA) mockEventHandler.events.get(1);
- assertEquals(priority, assignEvent.getPriority());
- assertEquals(mockAttemptId, assignEvent.getTaskAttemptId());
- }
-
- @Test (timeout = 5000)
- public void testTaskBasedAffinity() throws Exception {
- Configuration conf = new Configuration(false);
- schedulerHandler.init(conf);
- schedulerHandler.start();
-
- TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
- TezTaskAttemptID taId = mock(TezTaskAttemptID.class);
- String affVertexName = "srcVertex";
- int affTaskIndex = 1;
- TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(affVertexName, affTaskIndex);
- VertexImpl affVertex = mock(VertexImpl.class);
- TaskImpl affTask = mock(TaskImpl.class);
- TaskAttemptImpl affAttempt = mock(TaskAttemptImpl.class);
- ContainerId affCId = mock(ContainerId.class);
- when(affVertex.getTotalTasks()).thenReturn(2);
- when(affVertex.getTask(affTaskIndex)).thenReturn(affTask);
- when(affTask.getSuccessfulAttempt()).thenReturn(affAttempt);
- when(affAttempt.getAssignedContainerID()).thenReturn(affCId);
- when(mockAppContext.getCurrentDAG().getVertex(affVertexName)).thenReturn(affVertex);
- Resource resource = Resource.newInstance(100, 1);
- AMSchedulerEventTALaunchRequest event = new AMSchedulerEventTALaunchRequest
- (taId, resource, null, mockTaskAttempt, locHint, 3, null, 0, 0, 0);
- schedulerHandler.notify.set(false);
- schedulerHandler.handle(event);
- synchronized (schedulerHandler.notify) {
- while (!schedulerHandler.notify.get()) {
- schedulerHandler.notify.wait();
- }
- }
-
- // verify mockTaskAttempt affinitized to expected affCId
- verify(mockTaskScheduler, times(1)).allocateTask(mockTaskAttempt, resource, affCId,
- Priority.newInstance(3), null, event);
-
- schedulerHandler.stop();
- schedulerHandler.close();
- }
-
- @Test (timeout = 5000)
- public void testContainerPreempted() throws IOException {
- Configuration conf = new Configuration(false);
- schedulerHandler.init(conf);
- schedulerHandler.start();
-
- String diagnostics = "Container preempted by RM.";
- TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
- ContainerStatus mockStatus = mock(ContainerStatus.class);
- ContainerId mockCId = mock(ContainerId.class);
- AMContainer mockAMContainer = mock(AMContainer.class);
- when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
- when(mockAMContainer.getContainerId()).thenReturn(mockCId);
- when(mockStatus.getContainerId()).thenReturn(mockCId);
- when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
- when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.PREEMPTED);
- schedulerHandler.containerCompleted(0, mockTask, mockStatus);
- assertEquals(1, mockEventHandler.events.size());
- Event event = mockEventHandler.events.get(0);
- assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
- AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
- assertEquals(mockCId, completedEvent.getContainerId());
- assertEquals("Container preempted externally. Container preempted by RM.",
- completedEvent.getDiagnostics());
- assertTrue(completedEvent.isPreempted());
- assertEquals(TaskAttemptTerminationCause.EXTERNAL_PREEMPTION,
- completedEvent.getTerminationCause());
- Assert.assertFalse(completedEvent.isDiskFailed());
-
- schedulerHandler.stop();
- schedulerHandler.close();
- }
-
- @Test (timeout = 5000)
- public void testContainerInternalPreempted() throws IOException {
- Configuration conf = new Configuration(false);
- schedulerHandler.init(conf);
- schedulerHandler.start();
-
- AMContainer mockAmContainer = mock(AMContainer.class);
- when(mockAmContainer.getTaskSchedulerIdentifier()).thenReturn(0);
- when(mockAmContainer.getContainerLauncherIdentifier()).thenReturn(0);
- when(mockAmContainer.getTaskCommunicatorIdentifier()).thenReturn(0);
- ContainerId mockCId = mock(ContainerId.class);
- verify(mockTaskScheduler, times(0)).deallocateContainer((ContainerId) any());
- when(mockAMContainerMap.get(mockCId)).thenReturn(mockAmContainer);
- schedulerHandler.preemptContainer(0, mockCId);
- verify(mockTaskScheduler, times(1)).deallocateContainer(mockCId);
- assertEquals(1, mockEventHandler.events.size());
- Event event = mockEventHandler.events.get(0);
- assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
- AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
- assertEquals(mockCId, completedEvent.getContainerId());
- assertEquals("Container preempted internally", completedEvent.getDiagnostics());
- assertTrue(completedEvent.isPreempted());
- Assert.assertFalse(completedEvent.isDiskFailed());
- assertEquals(TaskAttemptTerminationCause.INTERNAL_PREEMPTION,
- completedEvent.getTerminationCause());
-
- schedulerHandler.stop();
- schedulerHandler.close();
- }
-
- @Test (timeout = 5000)
- public void testContainerDiskFailed() throws IOException {
- Configuration conf = new Configuration(false);
- schedulerHandler.init(conf);
- schedulerHandler.start();
-
- String diagnostics = "NM disk failed.";
- TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
- ContainerStatus mockStatus = mock(ContainerStatus.class);
- ContainerId mockCId = mock(ContainerId.class);
- AMContainer mockAMContainer = mock(AMContainer.class);
- when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
- when(mockAMContainer.getContainerId()).thenReturn(mockCId);
- when(mockStatus.getContainerId()).thenReturn(mockCId);
- when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
- when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.DISKS_FAILED);
- schedulerHandler.containerCompleted(0, mockTask, mockStatus);
- assertEquals(1, mockEventHandler.events.size());
- Event event = mockEventHandler.events.get(0);
- assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
- AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
- assertEquals(mockCId, completedEvent.getContainerId());
- assertEquals("Container disk failed. NM disk failed.",
- completedEvent.getDiagnostics());
- Assert.assertFalse(completedEvent.isPreempted());
- assertTrue(completedEvent.isDiskFailed());
- assertEquals(TaskAttemptTerminationCause.NODE_DISK_ERROR,
- completedEvent.getTerminationCause());
-
- schedulerHandler.stop();
- schedulerHandler.close();
- }
-
- @Test (timeout = 5000)
- public void testContainerExceededPMem() throws IOException {
- Configuration conf = new Configuration(false);
- schedulerHandler.init(conf);
- schedulerHandler.start();
-
- String diagnostics = "Exceeded Physical Memory";
- TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
- ContainerStatus mockStatus = mock(ContainerStatus.class);
- ContainerId mockCId = mock(ContainerId.class);
- AMContainer mockAMContainer = mock(AMContainer.class);
- when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
- when(mockAMContainer.getContainerId()).thenReturn(mockCId);
- when(mockStatus.getContainerId()).thenReturn(mockCId);
- when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
- // use -104 rather than ContainerExitStatus.KILLED_EXCEEDED_PMEM because
- // ContainerExitStatus.KILLED_EXCEEDED_PMEM is only available after hadoop-2.5
- when(mockStatus.getExitStatus()).thenReturn(-104);
- schedulerHandler.containerCompleted(0, mockTask, mockStatus);
- assertEquals(1, mockEventHandler.events.size());
- Event event = mockEventHandler.events.get(0);
- assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
- AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
- assertEquals(mockCId, completedEvent.getContainerId());
- assertEquals("Container failed, exitCode=-104. Exceeded Physical Memory",
- completedEvent.getDiagnostics());
- Assert.assertFalse(completedEvent.isPreempted());
- Assert.assertFalse(completedEvent.isDiskFailed());
- assertEquals(TaskAttemptTerminationCause.CONTAINER_EXITED,
- completedEvent.getTerminationCause());
-
- schedulerHandler.stop();
- schedulerHandler.close();
- }
-
- @Test (timeout = 5000)
- public void testHistoryUrlConf() throws Exception {
- Configuration conf = schedulerHandler.appContext.getAMConf();
-
- // ensure history url is empty when timeline server is not the logging class
- conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9999");
- assertTrue("".equals(schedulerHandler.getHistoryUrl()));
-
- // ensure expansion of url happens
- conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
- "org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService");
- final ApplicationId mockApplicationId = mock(ApplicationId.class);
- doReturn("TEST_APP_ID").when(mockApplicationId).toString();
- doReturn(mockApplicationId).when(mockAppContext).getApplicationID();
- assertTrue("http://ui-host:9999/#/tez-app/TEST_APP_ID"
- .equals(schedulerHandler.getHistoryUrl()));
-
- // ensure the trailing / in history url is handled
- conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9998/");
- assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
- .equals(schedulerHandler.getHistoryUrl()));
-
- // ensure missing scheme in history url is handled
- conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/");
- Assert.assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
- .equals(schedulerHandler.getHistoryUrl()));
-
- // handle bad template ex without begining /
- conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
- "__HISTORY_URL_BASE__#/somepath");
- assertTrue("http://ui-host:9998/#/somepath"
- .equals(schedulerHandler.getHistoryUrl()));
-
- conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
- "__HISTORY_URL_BASE__?viewPath=tez-app/__APPLICATION_ID__");
- conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://localhost/ui/tez");
- assertTrue("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID"
- .equals(schedulerHandler.getHistoryUrl()));
-
- }
-
- @Test(timeout = 5000)
- public void testNoSchedulerSpecified() throws IOException {
- try {
- new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
- mockSigMatcher, mockWebUIService, null, false);
- fail("Expecting an IllegalStateException with no schedulers specified");
- } catch (IllegalArgumentException e) {
- }
- }
-
- // Verified via statics
- @Test(timeout = 5000)
- public void testCustomTaskSchedulerSetup() throws IOException {
- Configuration conf = new Configuration(false);
- conf.set("testkey", "testval");
- UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
-
- String customSchedulerName = "fakeScheduler";
- List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
- ByteBuffer bb = ByteBuffer.allocate(4);
- bb.putInt(0, 3);
- UserPayload userPayload = UserPayload.create(bb);
- taskSchedulers.add(
- new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName())
- .setUserPayload(userPayload));
- taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
- .setUserPayload(defaultPayload));
-
- TSEHForMultipleSchedulersTest tseh =
- new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
- mockSigMatcher, mockWebUIService, taskSchedulers, false);
-
- tseh.init(conf);
- tseh.start();
-
- // Verify that the YARN task scheduler is installed by default
- assertTrue(tseh.getYarnSchedulerCreated());
- assertFalse(tseh.getUberSchedulerCreated());
- assertEquals(2, tseh.getNumCreateInvocations());
-
- // Verify the order of the schedulers
- assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
- assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));
-
- // Verify the payload setup for the custom task scheduler
- assertNotNull(tseh.getTaskSchedulerContext(0));
- assertEquals(bb, tseh.getTaskSchedulerContext(0).getInitialUserPayload().getPayload());
-
- // Verify the payload on the yarn scheduler
- assertNotNull(tseh.getTaskSchedulerContext(1));
- Configuration parsed = TezUtils.createConfFromUserPayload(tseh.getTaskSchedulerContext(1).getInitialUserPayload());
- assertEquals("testval", parsed.get("testkey"));
- }
-
- @Test(timeout = 5000)
- public void testTaskSchedulerRouting() throws Exception {
- Configuration conf = new Configuration(false);
- UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
-
- String customSchedulerName = "fakeScheduler";
- List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
- ByteBuffer bb = ByteBuffer.allocate(4);
- bb.putInt(0, 3);
- UserPayload userPayload = UserPayload.create(bb);
- taskSchedulers.add(
- new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName())
- .setUserPayload(userPayload));
- taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
- .setUserPayload(defaultPayload));
-
- TSEHForMultipleSchedulersTest tseh =
- new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
- mockSigMatcher, mockWebUIService, taskSchedulers, false);
-
- tseh.init(conf);
- tseh.start();
-
- // Verify that the YARN task scheduler is installed by default
- assertTrue(tseh.getYarnSchedulerCreated());
- assertFalse(tseh.getUberSchedulerCreated());
- assertEquals(2, tseh.getNumCreateInvocations());
-
- // Verify the order of the schedulers
- assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
- assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));
-
- verify(tseh.getTestTaskScheduler(0)).initialize();
- verify(tseh.getTestTaskScheduler(0)).start();
-
- ApplicationId appId = ApplicationId.newInstance(1000, 1);
- TezDAGID dagId = TezDAGID.getInstance(appId, 1);
- TezVertexID vertexID = TezVertexID.getInstance(dagId, 1);
- TezTaskID taskId1 = TezTaskID.getInstance(vertexID, 1);
- TezTaskAttemptID attemptId11 = TezTaskAttemptID.getInstance(taskId1, 1);
- TezTaskID taskId2 = TezTaskID.getInstance(vertexID, 2);
- TezTaskAttemptID attemptId21 = TezTaskAttemptID.getInstance(taskId2, 1);
-
- Resource resource = Resource.newInstance(1024, 1);
-
- TaskAttempt mockTaskAttempt1 = mock(TaskAttempt.class);
- TaskAttempt mockTaskAttempt2 = mock(TaskAttempt.class);
-
- AMSchedulerEventTALaunchRequest launchRequest1 =
- new AMSchedulerEventTALaunchRequest(attemptId11, resource, mock(TaskSpec.class),
- mockTaskAttempt1, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 0, 0,
- 0);
-
- tseh.handle(launchRequest1);
-
- verify(tseh.getTestTaskScheduler(0)).allocateTask(eq(mockTaskAttempt1), eq(resource),
- any(String[].class), any(String[].class), any(Priority.class), any(Object.class),
- eq(launchRequest1));
-
- AMSchedulerEventTALaunchRequest launchRequest2 =
- new AMSchedulerEventTALaunchRequest(attemptId21, resource, mock(TaskSpec.class),
- mockTaskAttempt2, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 1, 0,
- 0);
- tseh.handle(launchRequest2);
- verify(tseh.getTestTaskScheduler(1)).allocateTask(eq(mockTaskAttempt2), eq(resource),
- any(String[].class), any(String[].class), any(Priority.class), any(Object.class),
- eq(launchRequest2));
- }
-
- private static class TSEHForMultipleSchedulersTest extends TaskSchedulerEventHandler {
-
- private final TaskScheduler yarnTaskScheduler;
- private final TaskScheduler uberTaskScheduler;
- private final AtomicBoolean uberSchedulerCreated = new AtomicBoolean(false);
- private final AtomicBoolean yarnSchedulerCreated = new AtomicBoolean(false);
- private final AtomicInteger numCreateInvocations = new AtomicInteger(0);
- private final Set<Integer> seenSchedulers = new HashSet<>();
- private final List<TaskSchedulerContext> taskSchedulerContexts = new LinkedList<>();
- private final List<String> taskSchedulerNames = new LinkedList<>();
- private final List<TaskScheduler> testTaskSchedulers = new LinkedList<>();
-
- public TSEHForMultipleSchedulersTest(AppContext appContext,
- DAGClientServer clientService,
- EventHandler eventHandler,
- ContainerSignatureMatcher containerSignatureMatcher,
- WebUIService webUI,
- List<NamedEntityDescriptor> schedulerDescriptors,
- boolean isPureLocalMode) {
- super(appContext, clientService, eventHandler, containerSignatureMatcher, webUI,
- schedulerDescriptors, isPureLocalMode);
- yarnTaskScheduler = mock(TaskScheduler.class);
- uberTaskScheduler = mock(TaskScheduler.class);
- }
-
- @Override
- TaskScheduler createTaskScheduler(String host, int port, String trackingUrl,
- AppContext appContext,
- NamedEntityDescriptor taskSchedulerDescriptor,
- long customAppIdIdentifier,
- int schedulerId) {
-
- numCreateInvocations.incrementAndGet();
- boolean added = seenSchedulers.add(schedulerId);
- assertTrue("Cannot add multiple schedulers with the same schedulerId", added);
- taskSchedulerNames.add(taskSchedulerDescriptor.getEntityName());
- return super.createTaskScheduler(host, port, trackingUrl, appContext, taskSchedulerDescriptor,
- customAppIdIdentifier, schedulerId);
- }
-
- @Override
- TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
- // Avoid wrapping in threads
- return rawContext;
- }
-
- @Override
- TaskScheduler createYarnTaskScheduler(TaskSchedulerContext taskSchedulerContext, int schedulerId) {
- taskSchedulerContexts.add(taskSchedulerContext);
- testTaskSchedulers.add(yarnTaskScheduler);
- yarnSchedulerCreated.set(true);
- return yarnTaskScheduler;
- }
-
- @Override
- TaskScheduler createUberTaskScheduler(TaskSchedulerContext taskSchedulerContext, int schedulerId) {
- taskSchedulerContexts.add(taskSchedulerContext);
- uberSchedulerCreated.set(true);
- testTaskSchedulers.add(yarnTaskScheduler);
- return uberTaskScheduler;
- }
-
- @Override
- TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext,
- NamedEntityDescriptor taskSchedulerDescriptor, int schedulerId) {
- taskSchedulerContexts.add(taskSchedulerContext);
- TaskScheduler taskScheduler = spy(super.createCustomTaskScheduler(taskSchedulerContext, taskSchedulerDescriptor, schedulerId));
- testTaskSchedulers.add(taskScheduler);
- return taskScheduler;
- }
-
- @Override
- // Inline handling of events.
- public void handle(AMSchedulerEvent event) {
- handleEvent(event);
- }
-
- public boolean getUberSchedulerCreated() {
- return uberSchedulerCreated.get();
- }
-
- public boolean getYarnSchedulerCreated() {
- return yarnSchedulerCreated.get();
- }
-
- public int getNumCreateInvocations() {
- return numCreateInvocations.get();
- }
-
- public TaskSchedulerContext getTaskSchedulerContext(int schedulerId) {
- return taskSchedulerContexts.get(schedulerId);
- }
-
- public String getTaskSchedulerName(int schedulerId) {
- return taskSchedulerNames.get(schedulerId);
- }
-
- public TaskScheduler getTestTaskScheduler(int schedulerId) {
- return testTaskSchedulers.get(schedulerId);
- }
- }
-
- public static class FakeTaskScheduler extends TaskScheduler {
-
- public FakeTaskScheduler(
- TaskSchedulerContext taskSchedulerContext) {
- super(taskSchedulerContext);
- }
-
- @Override
- public Resource getAvailableResources() {
- return null;
- }
-
- @Override
- public int getClusterNodeCount() {
- return 0;
- }
-
- @Override
- public void dagComplete() {
-
- }
-
- @Override
- public Resource getTotalResources() {
- return null;
- }
-
- @Override
- public void blacklistNode(NodeId nodeId) {
-
- }
-
- @Override
- public void unblacklistNode(NodeId nodeId) {
-
- }
-
- @Override
- public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks,
- Priority priority, Object containerSignature, Object clientCookie) {
-
- }
-
- @Override
- public void allocateTask(Object task, Resource capability, ContainerId containerId,
- Priority priority, Object containerSignature, Object clientCookie) {
-
- }
-
- @Override
- public boolean deallocateTask(Object task, boolean taskSucceeded,
- TaskAttemptEndReason endReason,
- String diagnostics) {
- return false;
- }
-
- @Override
- public Object deallocateContainer(ContainerId containerId) {
- return null;
- }
-
- @Override
- public void setShouldUnregister() {
-
- }
-
- @Override
- public boolean hasUnregistered() {
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
index c13ca5a..b1bc491 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java
@@ -125,19 +125,19 @@ class TestTaskSchedulerHelpers {
}
// Overrides start / stop. Will be controlled without the extra event handling thread.
- static class TaskSchedulerEventHandlerForTest extends
- TaskSchedulerEventHandler {
+ static class TaskSchedulerManagerForTest extends
+ TaskSchedulerManager {
private TezAMRMClientAsync<CookieContainerRequest> amrmClientAsync;
private ContainerSignatureMatcher containerSignatureMatcher;
private UserPayload defaultPayload;
@SuppressWarnings("rawtypes")
- public TaskSchedulerEventHandlerForTest(AppContext appContext,
- EventHandler eventHandler,
- TezAMRMClientAsync<CookieContainerRequest> amrmClientAsync,
- ContainerSignatureMatcher containerSignatureMatcher,
- UserPayload defaultPayload) {
+ public TaskSchedulerManagerForTest(AppContext appContext,
+ EventHandler eventHandler,
+ TezAMRMClientAsync<CookieContainerRequest> amrmClientAsync,
+ ContainerSignatureMatcher containerSignatureMatcher,
+ UserPayload defaultPayload) {
super(appContext, null, eventHandler, containerSignatureMatcher, null,
Lists.newArrayList(new NamedEntityDescriptor("FakeScheduler", null)),
false);
[6/7] tez git commit: TEZ-2708. Rename classes and variables post
TEZ-2003 changes. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
deleted file mode 100644
index 07d269d..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ /dev/null
@@ -1,410 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.launcher;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
-import org.apache.tez.serviceplugins.api.ContainerLauncher;
-import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
-import org.apache.tez.serviceplugins.api.ContainerStopRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy;
-import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezUncheckedException;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-
-// TODO See what part of this lifecycle and state management can be simplified.
-// Ideally, no state - only sendStart / sendStop.
-
-// TODO Review this entire code and clean it up.
-
-/**
- * This class is responsible for launching of containers.
- */
-public class ContainerLauncherImpl extends ContainerLauncher {
-
- // TODO Ensure the same thread is used to launch / stop the same container. Or - ensure event ordering.
- static final Logger LOG = LoggerFactory.getLogger(ContainerLauncherImpl.class);
-
- private final ConcurrentHashMap<ContainerId, Container> containers =
- new ConcurrentHashMap<>();
- protected ThreadPoolExecutor launcherPool;
- protected static final int INITIAL_POOL_SIZE = 10;
- private final int limitOnPoolSize;
- private final Configuration conf;
- private Thread eventHandlingThread;
- protected BlockingQueue<ContainerOp> eventQueue = new LinkedBlockingQueue<>();
- private ContainerManagementProtocolProxy cmProxy;
- private AtomicBoolean serviceStopped = new AtomicBoolean(false);
-
- private Container getContainer(ContainerOp event) {
- ContainerId id = event.getBaseOperation().getContainerId();
- Container c = containers.get(id);
- if(c == null) {
- c = new Container(id,
- event.getBaseOperation().getNodeId().toString(), event.getBaseOperation().getContainerToken());
- Container old = containers.putIfAbsent(id, c);
- if(old != null) {
- c = old;
- }
- }
- return c;
- }
-
- private void removeContainerIfDone(ContainerId id) {
- Container c = containers.get(id);
- if(c != null && c.isCompletelyDone()) {
- containers.remove(id);
- }
- }
-
-
- private static enum ContainerState {
- PREP, FAILED, RUNNING, DONE, KILLED_BEFORE_LAUNCH
- }
-
- private class Container {
- private ContainerState state;
- // store enough information to be able to cleanup the container
- private ContainerId containerID;
- final private String containerMgrAddress;
- private Token containerToken;
-
- public Container(ContainerId containerID,
- String containerMgrAddress, Token containerToken) {
- this.state = ContainerState.PREP;
- this.containerMgrAddress = containerMgrAddress;
- this.containerID = containerID;
- this.containerToken = containerToken;
- }
-
- public synchronized boolean isCompletelyDone() {
- return state == ContainerState.DONE || state == ContainerState.FAILED;
- }
-
- @SuppressWarnings("unchecked")
- public synchronized void launch(ContainerLaunchRequest event) {
- LOG.info("Launching Container with Id: " + event.getContainerId());
- if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
- state = ContainerState.DONE;
- sendContainerLaunchFailedMsg(event.getContainerId(),
- "Container was killed before it was launched");
- return;
- }
-
- ContainerManagementProtocolProxyData proxy = null;
- try {
-
- proxy = getCMProxy(containerID, containerMgrAddress,
- containerToken);
-
- // Construct the actual Container
- ContainerLaunchContext containerLaunchContext =
- event.getContainerLaunchContext();
-
- // Now launch the actual container
- StartContainerRequest startRequest = Records
- .newRecord(StartContainerRequest.class);
- startRequest.setContainerToken(event.getContainerToken());
- startRequest.setContainerLaunchContext(containerLaunchContext);
-
- StartContainersResponse response =
- proxy.getContainerManagementProtocol().startContainers(
- StartContainersRequest.newInstance(
- Collections.singletonList(startRequest)));
- if (response.getFailedRequests() != null
- && !response.getFailedRequests().isEmpty()) {
- throw response.getFailedRequests().get(containerID).deSerialize();
- }
-
- // after launching, send launched event to task attempt to move
- // it from ASSIGNED to RUNNING state
- getContext().containerLaunched(containerID);
- this.state = ContainerState.RUNNING;
- } catch (Throwable t) {
- String message = "Container launch failed for " + containerID + " : "
- + ExceptionUtils.getStackTrace(t);
- this.state = ContainerState.FAILED;
- sendContainerLaunchFailedMsg(containerID, message);
- } finally {
- if (proxy != null) {
- cmProxy.mayBeCloseProxy(proxy);
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- public synchronized void kill() {
-
- if(isCompletelyDone()) {
- return;
- }
- if(this.state == ContainerState.PREP) {
- this.state = ContainerState.KILLED_BEFORE_LAUNCH;
- } else {
- LOG.info("Sending a stop request to the NM for ContainerId: "
- + containerID);
-
- ContainerManagementProtocolProxyData proxy = null;
- try {
- proxy = getCMProxy(this.containerID, this.containerMgrAddress,
- this.containerToken);
-
- // kill the remote container if already launched
- StopContainersRequest stopRequest = Records
- .newRecord(StopContainersRequest.class);
- stopRequest.setContainerIds(Collections.singletonList(containerID));
-
- proxy.getContainerManagementProtocol().stopContainers(stopRequest);
-
- // If stopContainer returns without an error, assuming the stop made
- // it over to the NodeManager.
- getContext().containerStopRequested(containerID);
- } catch (Throwable t) {
-
- // ignore the cleanup failure
- String message = "cleanup failed for container "
- + this.containerID + " : "
- + ExceptionUtils.getStackTrace(t);
- getContext().containerStopFailed(containerID, message);
- LOG.warn(message);
- this.state = ContainerState.DONE;
- return;
- } finally {
- if (proxy != null) {
- cmProxy.mayBeCloseProxy(proxy);
- }
- }
- this.state = ContainerState.DONE;
- }
- }
- }
-
- public ContainerLauncherImpl(ContainerLauncherContext containerLauncherContext) {
- super(containerLauncherContext);
- try {
- this.conf = TezUtils.createConfFromUserPayload(containerLauncherContext.getInitialUserPayload());
- } catch (IOException e) {
- throw new TezUncheckedException(
- "Failed to parse user payload for " + ContainerLauncherImpl.class.getSimpleName(), e);
- }
- conf.setInt(
- CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
- 0);
- this.limitOnPoolSize = conf.getInt(
- TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
- TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT_DEFAULT);
- LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize);
- }
-
- @Override
- public void start() {
- // pass a copy of config to ContainerManagementProtocolProxy until YARN-3497 is fixed
- cmProxy =
- new ContainerManagementProtocolProxy(conf);
-
- ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
- "ContainerLauncher #%d").setDaemon(true).build();
-
- // Start with a default core-pool size of 10 and change it dynamically.
- launcherPool = new ThreadPoolExecutor(INITIAL_POOL_SIZE,
- Integer.MAX_VALUE, 1, TimeUnit.HOURS,
- new LinkedBlockingQueue<Runnable>(),
- tf, new CustomizedRejectedExecutionHandler());
- eventHandlingThread = new Thread() {
- @Override
- public void run() {
- ContainerOp event = null;
- while (!Thread.currentThread().isInterrupted()) {
- try {
- event = eventQueue.take();
- } catch (InterruptedException e) {
- if(!serviceStopped.get()) {
- LOG.error("Returning, interrupted : " + e);
- }
- return;
- }
- int poolSize = launcherPool.getCorePoolSize();
-
- // See if we need up the pool size only if haven't reached the
- // maximum limit yet.
- if (poolSize != limitOnPoolSize) {
-
- // nodes where containers will run at *this* point of time. This is
- // *not* the cluster size and doesn't need to be.
- int numNodes =
- getContext().getNumNodes(TezConstants.getTezYarnServicePluginName());
- int idealPoolSize = Math.min(limitOnPoolSize, numNodes);
-
- if (poolSize < idealPoolSize) {
- // Bump up the pool size to idealPoolSize+INITIAL_POOL_SIZE, the
- // later is just a buffer so we are not always increasing the
- // pool-size
- int newPoolSize = Math.min(limitOnPoolSize, idealPoolSize
- + INITIAL_POOL_SIZE);
- LOG.info("Setting ContainerLauncher pool size to " + newPoolSize
- + " as number-of-nodes to talk to is " + numNodes);
- launcherPool.setCorePoolSize(newPoolSize);
- }
- }
-
- // the events from the queue are handled in parallel
- // using a thread pool
- launcherPool.execute(createEventProcessor(event));
-
- // TODO: Group launching of multiple containers to a single
- // NodeManager into a single connection
- }
- }
- };
- eventHandlingThread.setName("ContainerLauncher Event Handler");
- eventHandlingThread.start();
- }
-
- private void shutdownAllContainers() {
- for (Container ct : this.containers.values()) {
- if (ct != null) {
- ct.kill();
- }
- }
- }
-
- @Override
- public void shutdown() {
- if(!serviceStopped.compareAndSet(false, true)) {
- LOG.info("Ignoring multiple stops");
- return;
- }
- // shutdown any containers that might be left running
- shutdownAllContainers();
- if (eventHandlingThread != null) {
- eventHandlingThread.interrupt();
- }
- if (launcherPool != null) {
- launcherPool.shutdownNow();
- }
- }
-
- protected EventProcessor createEventProcessor(ContainerOp event) {
- return new EventProcessor(event);
- }
-
- protected ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData getCMProxy(
- ContainerId containerID, final String containerManagerBindAddr,
- Token containerToken) throws IOException {
- return cmProxy.getProxy(containerManagerBindAddr, containerID);
- }
-
- /**
- * Setup and start the container on remote nodemanager.
- */
- class EventProcessor implements Runnable {
- private ContainerOp event;
-
- EventProcessor(ContainerOp event) {
- this.event = event;
- }
-
- @Override
- public void run() {
- LOG.info("Processing operation {}", event.toString());
-
- // Load ContainerManager tokens before creating a connection.
- // TODO: Do it only once per NodeManager.
- ContainerId containerID = event.getBaseOperation().getContainerId();
-
- Container c = getContainer(event);
- switch(event.getOpType()) {
- case LAUNCH_REQUEST:
- ContainerLaunchRequest launchRequest = event.getLaunchRequest();
- c.launch(launchRequest);
- break;
- case STOP_REQUEST:
- c.kill();
- break;
- }
- removeContainerIfDone(containerID);
- }
- }
-
- /**
- * ThreadPoolExecutor.submit may fail if you are submitting task
- * when ThreadPoolExecutor is shutting down (DAGAppMaster is shutting down).
- * Use this CustomizedRejectedExecutionHandler to just logging rather than abort the application.
- */
- private static class CustomizedRejectedExecutionHandler implements RejectedExecutionHandler {
- @Override
- public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
- LOG.warn("Can't submit task to ThreadPoolExecutor:" + executor);
- }
- }
-
- @SuppressWarnings("unchecked")
- void sendContainerLaunchFailedMsg(ContainerId containerId,
- String message) {
- LOG.error(message);
- getContext().containerLaunchFailed(containerId, message);
- }
-
-
- @Override
- public void launchContainer(ContainerLaunchRequest launchRequest) {
- try {
- eventQueue.put(new ContainerOp(ContainerOp.OPType.LAUNCH_REQUEST, launchRequest));
- } catch (InterruptedException e) {
- throw new TezUncheckedException(e);
- }
- }
-
- @Override
- public void stopContainer(ContainerStopRequest stopRequest) {
- try {
- eventQueue.put(new ContainerOp(ContainerOp.OPType.STOP_REQUEST, stopRequest));
- } catch (InterruptedException e) {
- throw new TezUncheckedException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java
new file mode 100644
index 0000000..15a10bd
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.launcher;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.UnknownHostException;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
+import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
+import org.apache.tez.serviceplugins.api.ContainerLauncher;
+import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
+import org.apache.tez.serviceplugins.api.ContainerStopRequest;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.ContainerLauncherContextImpl;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.rm.ContainerLauncherEvent;
+import org.apache.tez.dag.app.rm.ContainerLauncherLaunchRequestEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ContainerLauncherManager extends AbstractService
+ implements EventHandler<ContainerLauncherEvent> {
+
+ static final Logger LOG = LoggerFactory.getLogger(TezContainerLauncherImpl.class);
+
+ @VisibleForTesting
+ final ContainerLauncher containerLaunchers[];
+ @VisibleForTesting
+ final ContainerLauncherContext containerLauncherContexts[];
+ protected final ServicePluginLifecycleAbstractService[] containerLauncherServiceWrappers;
+ private final AppContext appContext;
+
+ @VisibleForTesting
+ public ContainerLauncherManager(ContainerLauncher containerLauncher, AppContext context) {
+ super(ContainerLauncherManager.class.getName());
+ this.appContext = context;
+ containerLaunchers = new ContainerLauncher[] {containerLauncher};
+ containerLauncherContexts = new ContainerLauncherContext[] {containerLauncher.getContext()};
+ containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[]{
+ new ServicePluginLifecycleAbstractService<>(containerLauncher)};
+ }
+
+ // Accepting conf to setup final parameters, if required.
+ public ContainerLauncherManager(AppContext context,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+ String workingDirectory,
+ List<NamedEntityDescriptor> containerLauncherDescriptors,
+ boolean isPureLocalMode) {
+ super(ContainerLauncherManager.class.getName());
+
+ this.appContext = context;
+ Preconditions.checkArgument(
+ containerLauncherDescriptors != null && !containerLauncherDescriptors.isEmpty(),
+ "ContainerLauncherDescriptors must be specified");
+ containerLauncherContexts = new ContainerLauncherContext[containerLauncherDescriptors.size()];
+ containerLaunchers = new ContainerLauncher[containerLauncherDescriptors.size()];
+ containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[containerLauncherDescriptors.size()];
+
+
+ for (int i = 0; i < containerLauncherDescriptors.size(); i++) {
+ UserPayload userPayload = containerLauncherDescriptors.get(i).getUserPayload();
+ ContainerLauncherContext containerLauncherContext =
+ new ContainerLauncherContextImpl(context, taskCommunicatorManagerInterface, userPayload);
+ containerLauncherContexts[i] = containerLauncherContext;
+ containerLaunchers[i] = createContainerLauncher(containerLauncherDescriptors.get(i), context,
+ containerLauncherContext, taskCommunicatorManagerInterface, workingDirectory, i, isPureLocalMode);
+ containerLauncherServiceWrappers[i] = new ServicePluginLifecycleAbstractService<>(containerLaunchers[i]);
+ }
+ }
+
+ @VisibleForTesting
+ ContainerLauncher createContainerLauncher(
+ NamedEntityDescriptor containerLauncherDescriptor,
+ AppContext context,
+ ContainerLauncherContext containerLauncherContext,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+ String workingDirectory,
+ int containerLauncherIndex,
+ boolean isPureLocalMode) {
+ if (containerLauncherDescriptor.getEntityName().equals(
+ TezConstants.getTezYarnServicePluginName())) {
+ return createYarnContainerLauncher(containerLauncherContext);
+ } else if (containerLauncherDescriptor.getEntityName()
+ .equals(TezConstants.getTezUberServicePluginName())) {
+ return createUberContainerLauncher(containerLauncherContext, context,
+ taskCommunicatorManagerInterface,
+ workingDirectory, isPureLocalMode);
+ } else {
+ return createCustomContainerLauncher(containerLauncherContext, containerLauncherDescriptor);
+ }
+ }
+
+ @VisibleForTesting
+ ContainerLauncher createYarnContainerLauncher(ContainerLauncherContext containerLauncherContext) {
+ LOG.info("Creating DefaultContainerLauncher");
+ return new TezContainerLauncherImpl(containerLauncherContext);
+ }
+
+ @VisibleForTesting
+ ContainerLauncher createUberContainerLauncher(ContainerLauncherContext containerLauncherContext,
+ AppContext context,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+ String workingDirectory,
+ boolean isPureLocalMode) {
+ LOG.info("Creating LocalContainerLauncher");
+ // TODO Post TEZ-2003. LocalContainerLauncher is special cased, since it makes use of
+ // extensive internals which are only available at runtime. Will likely require
+ // some kind of runtime binding of parameters in the payload to work correctly.
+ try {
+ return
+ new LocalContainerLauncher(containerLauncherContext, context,
+ taskCommunicatorManagerInterface,
+ workingDirectory, isPureLocalMode);
+ } catch (UnknownHostException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+
+ @VisibleForTesting
+ @SuppressWarnings("unchecked")
+ ContainerLauncher createCustomContainerLauncher(ContainerLauncherContext containerLauncherContext,
+ NamedEntityDescriptor containerLauncherDescriptor) {
+ LOG.info("Creating container launcher {}:{} ", containerLauncherDescriptor.getEntityName(),
+ containerLauncherDescriptor.getClassName());
+ Class<? extends ContainerLauncher> containerLauncherClazz =
+ (Class<? extends ContainerLauncher>) ReflectionUtils.getClazz(
+ containerLauncherDescriptor.getClassName());
+ try {
+ Constructor<? extends ContainerLauncher> ctor = containerLauncherClazz
+ .getConstructor(ContainerLauncherContext.class);
+ return ctor.newInstance(containerLauncherContext);
+ } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
+ throw new TezUncheckedException(e);
+ }
+
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ for (int i = 0 ; i < containerLaunchers.length ; i++) {
+ containerLauncherServiceWrappers[i].init(conf);
+ }
+ }
+
+ @Override
+ public void serviceStart() {
+ for (int i = 0 ; i < containerLaunchers.length ; i++) {
+ containerLauncherServiceWrappers[i].start();
+ }
+ }
+
+ @Override
+ public void serviceStop() {
+ for (int i = 0 ; i < containerLaunchers.length ; i++) {
+ containerLauncherServiceWrappers[i].stop();
+ }
+ }
+
+ public void dagComplete(DAG dag) {
+ // Nothing required at the moment. Containers are shared across DAGs
+ }
+
+ public void dagSubmitted() {
+ // Nothing to do right now. Indicates that a new DAG has been submitted and
+ // the context has updated information.
+ }
+
+
+ @Override
+ public void handle(ContainerLauncherEvent event) {
+ int launcherId = event.getLauncherId();
+ String schedulerName = appContext.getTaskSchedulerName(event.getSchedulerId());
+ String taskCommName = appContext.getTaskCommunicatorName(event.getTaskCommId());
+ switch (event.getType()) {
+ case CONTAINER_LAUNCH_REQUEST:
+ ContainerLauncherLaunchRequestEvent launchEvent = (ContainerLauncherLaunchRequestEvent) event;
+ ContainerLaunchRequest launchRequest =
+ new ContainerLaunchRequest(launchEvent.getNodeId(), launchEvent.getContainerId(),
+ launchEvent.getContainerToken(), launchEvent.getContainerLaunchContext(),
+ launchEvent.getContainer(), schedulerName,
+ taskCommName);
+ containerLaunchers[launcherId].launchContainer(launchRequest);
+ break;
+ case CONTAINER_STOP_REQUEST:
+ ContainerStopRequest stopRequest =
+ new ContainerStopRequest(event.getNodeId(), event.getContainerId(),
+ event.getContainerToken(), schedulerName, taskCommName);
+ containerLaunchers[launcherId].stopContainer(stopRequest);
+ break;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
deleted file mode 100644
index b56bd5b..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherRouter.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.launcher;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.net.UnknownHostException;
-import java.util.List;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.common.ReflectionUtils;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
-import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
-import org.apache.tez.serviceplugins.api.ContainerLauncher;
-import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
-import org.apache.tez.serviceplugins.api.ContainerStopRequest;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.ContainerLauncherContextImpl;
-import org.apache.tez.dag.app.TaskAttemptListener;
-import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
-import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ContainerLauncherRouter extends AbstractService
- implements EventHandler<NMCommunicatorEvent> {
-
- static final Logger LOG = LoggerFactory.getLogger(ContainerLauncherImpl.class);
-
- @VisibleForTesting
- final ContainerLauncher containerLaunchers[];
- @VisibleForTesting
- final ContainerLauncherContext containerLauncherContexts[];
- protected final ServicePluginLifecycleAbstractService[] containerLauncherServiceWrappers;
- private final AppContext appContext;
-
- @VisibleForTesting
- public ContainerLauncherRouter(ContainerLauncher containerLauncher, AppContext context) {
- super(ContainerLauncherRouter.class.getName());
- this.appContext = context;
- containerLaunchers = new ContainerLauncher[] {containerLauncher};
- containerLauncherContexts = new ContainerLauncherContext[] {containerLauncher.getContext()};
- containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[]{
- new ServicePluginLifecycleAbstractService<>(containerLauncher)};
- }
-
- // Accepting conf to setup final parameters, if required.
- public ContainerLauncherRouter(AppContext context,
- TaskAttemptListener taskAttemptListener,
- String workingDirectory,
- List<NamedEntityDescriptor> containerLauncherDescriptors,
- boolean isPureLocalMode) {
- super(ContainerLauncherRouter.class.getName());
-
- this.appContext = context;
- Preconditions.checkArgument(
- containerLauncherDescriptors != null && !containerLauncherDescriptors.isEmpty(),
- "ContainerLauncherDescriptors must be specified");
- containerLauncherContexts = new ContainerLauncherContext[containerLauncherDescriptors.size()];
- containerLaunchers = new ContainerLauncher[containerLauncherDescriptors.size()];
- containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[containerLauncherDescriptors.size()];
-
-
- for (int i = 0; i < containerLauncherDescriptors.size(); i++) {
- UserPayload userPayload = containerLauncherDescriptors.get(i).getUserPayload();
- ContainerLauncherContext containerLauncherContext =
- new ContainerLauncherContextImpl(context, taskAttemptListener, userPayload);
- containerLauncherContexts[i] = containerLauncherContext;
- containerLaunchers[i] = createContainerLauncher(containerLauncherDescriptors.get(i), context,
- containerLauncherContext, taskAttemptListener, workingDirectory, i, isPureLocalMode);
- containerLauncherServiceWrappers[i] = new ServicePluginLifecycleAbstractService<>(containerLaunchers[i]);
- }
- }
-
- @VisibleForTesting
- ContainerLauncher createContainerLauncher(
- NamedEntityDescriptor containerLauncherDescriptor,
- AppContext context,
- ContainerLauncherContext containerLauncherContext,
- TaskAttemptListener taskAttemptListener,
- String workingDirectory,
- int containerLauncherIndex,
- boolean isPureLocalMode) {
- if (containerLauncherDescriptor.getEntityName().equals(
- TezConstants.getTezYarnServicePluginName())) {
- return createYarnContainerLauncher(containerLauncherContext);
- } else if (containerLauncherDescriptor.getEntityName()
- .equals(TezConstants.getTezUberServicePluginName())) {
- return createUberContainerLauncher(containerLauncherContext, context, taskAttemptListener,
- workingDirectory, isPureLocalMode);
- } else {
- return createCustomContainerLauncher(containerLauncherContext, containerLauncherDescriptor);
- }
- }
-
- @VisibleForTesting
- ContainerLauncher createYarnContainerLauncher(ContainerLauncherContext containerLauncherContext) {
- LOG.info("Creating DefaultContainerLauncher");
- return new ContainerLauncherImpl(containerLauncherContext);
- }
-
- @VisibleForTesting
- ContainerLauncher createUberContainerLauncher(ContainerLauncherContext containerLauncherContext,
- AppContext context,
- TaskAttemptListener taskAttemptListener,
- String workingDirectory,
- boolean isPureLocalMode) {
- LOG.info("Creating LocalContainerLauncher");
- // TODO Post TEZ-2003. LocalContainerLauncher is special cased, since it makes use of
- // extensive internals which are only available at runtime. Will likely require
- // some kind of runtime binding of parameters in the payload to work correctly.
- try {
- return
- new LocalContainerLauncher(containerLauncherContext, context, taskAttemptListener,
- workingDirectory, isPureLocalMode);
- } catch (UnknownHostException e) {
- throw new TezUncheckedException(e);
- }
- }
-
- @VisibleForTesting
- @SuppressWarnings("unchecked")
- ContainerLauncher createCustomContainerLauncher(ContainerLauncherContext containerLauncherContext,
- NamedEntityDescriptor containerLauncherDescriptor) {
- LOG.info("Creating container launcher {}:{} ", containerLauncherDescriptor.getEntityName(),
- containerLauncherDescriptor.getClassName());
- Class<? extends ContainerLauncher> containerLauncherClazz =
- (Class<? extends ContainerLauncher>) ReflectionUtils.getClazz(
- containerLauncherDescriptor.getClassName());
- try {
- Constructor<? extends ContainerLauncher> ctor = containerLauncherClazz
- .getConstructor(ContainerLauncherContext.class);
- return ctor.newInstance(containerLauncherContext);
- } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
- throw new TezUncheckedException(e);
- }
-
- }
-
- @Override
- public void serviceInit(Configuration conf) {
- for (int i = 0 ; i < containerLaunchers.length ; i++) {
- containerLauncherServiceWrappers[i].init(conf);
- }
- }
-
- @Override
- public void serviceStart() {
- for (int i = 0 ; i < containerLaunchers.length ; i++) {
- containerLauncherServiceWrappers[i].start();
- }
- }
-
- @Override
- public void serviceStop() {
- for (int i = 0 ; i < containerLaunchers.length ; i++) {
- containerLauncherServiceWrappers[i].stop();
- }
- }
-
- public void dagComplete(DAG dag) {
- // Nothing required at the moment. Containers are shared across DAGs
- }
-
- public void dagSubmitted() {
- // Nothing to do right now. Indicates that a new DAG has been submitted and
- // the context has updated information.
- }
-
-
- @Override
- public void handle(NMCommunicatorEvent event) {
- int launcherId = event.getLauncherId();
- String schedulerName = appContext.getTaskSchedulerName(event.getSchedulerId());
- String taskCommName = appContext.getTaskCommunicatorName(event.getTaskCommId());
- switch (event.getType()) {
- case CONTAINER_LAUNCH_REQUEST:
- NMCommunicatorLaunchRequestEvent launchEvent = (NMCommunicatorLaunchRequestEvent) event;
- ContainerLaunchRequest launchRequest =
- new ContainerLaunchRequest(launchEvent.getNodeId(), launchEvent.getContainerId(),
- launchEvent.getContainerToken(), launchEvent.getContainerLaunchContext(),
- launchEvent.getContainer(), schedulerName,
- taskCommName);
- containerLaunchers[launcherId].launchContainer(launchRequest);
- break;
- case CONTAINER_STOP_REQUEST:
- ContainerStopRequest stopRequest =
- new ContainerStopRequest(event.getNodeId(), event.getContainerId(),
- event.getContainerToken(), schedulerName, taskCommName);
- containerLaunchers[launcherId].stopContainer(stopRequest);
- break;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 1d3e6df..6cd6fce 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -63,7 +63,7 @@ import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
import org.apache.tez.runtime.api.ExecutionContext;
import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
@@ -83,7 +83,7 @@ public class LocalContainerLauncher extends ContainerLauncher {
private final AppContext context;
private final AtomicBoolean serviceStopped = new AtomicBoolean(false);
private final String workingDirectory;
- private final TaskAttemptListener tal;
+ private final TaskCommunicatorManagerInterface tal;
private final Map<String, String> localEnv;
private final ExecutionContext executionContext;
private final int numExecutors;
@@ -105,7 +105,7 @@ public class LocalContainerLauncher extends ContainerLauncher {
public LocalContainerLauncher(ContainerLauncherContext containerLauncherContext,
AppContext context,
- TaskAttemptListener taskAttemptListener,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
String workingDirectory,
boolean isPureLocalMode) throws UnknownHostException {
// TODO Post TEZ-2003. Most of this information is dynamic and only available after the AM
@@ -114,7 +114,7 @@ public class LocalContainerLauncher extends ContainerLauncher {
// after the AM starts up.
super(containerLauncherContext);
this.context = context;
- this.tal = taskAttemptListener;
+ this.tal = taskCommunicatorManagerInterface;
this.workingDirectory = workingDirectory;
this.isPureLocalMode = isPureLocalMode;
if (isPureLocalMode) {
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java
new file mode 100644
index 0000000..3556c51
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java
@@ -0,0 +1,410 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.app.launcher;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
+import org.apache.tez.serviceplugins.api.ContainerLauncher;
+import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
+import org.apache.tez.serviceplugins.api.ContainerStopRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy;
+import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezUncheckedException;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+
+// TODO See what part of this lifecycle and state management can be simplified.
+// Ideally, no state - only sendStart / sendStop.
+
+// TODO Review this entire code and clean it up.
+
+/**
+ * This class is responsible for launching of containers.
+ */
+public class TezContainerLauncherImpl extends ContainerLauncher {
+
+ // TODO Ensure the same thread is used to launch / stop the same container. Or - ensure event ordering.
+ static final Logger LOG = LoggerFactory.getLogger(TezContainerLauncherImpl.class);
+
+ private final ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>();
+ protected ThreadPoolExecutor launcherPool;
+ protected static final int INITIAL_POOL_SIZE = 10;
+ private final int limitOnPoolSize;
+ private final Configuration conf;
+ private Thread eventHandlingThread;
+ protected BlockingQueue<ContainerOp> eventQueue = new LinkedBlockingQueue<>();
+ private ContainerManagementProtocolProxy cmProxy;
+ private AtomicBoolean serviceStopped = new AtomicBoolean(false);
+
+ private Container getContainer(ContainerOp event) {
+ ContainerId id = event.getBaseOperation().getContainerId();
+ Container c = containers.get(id);
+ if(c == null) {
+ c = new Container(id,
+ event.getBaseOperation().getNodeId().toString(), event.getBaseOperation().getContainerToken());
+ Container old = containers.putIfAbsent(id, c);
+ if(old != null) {
+ c = old;
+ }
+ }
+ return c;
+ }
+
+ private void removeContainerIfDone(ContainerId id) {
+ Container c = containers.get(id);
+ if(c != null && c.isCompletelyDone()) {
+ containers.remove(id);
+ }
+ }
+
+
+ private static enum ContainerState {
+ PREP, FAILED, RUNNING, DONE, KILLED_BEFORE_LAUNCH
+ }
+
+ private class Container {
+ private ContainerState state;
+ // store enough information to be able to cleanup the container
+ private ContainerId containerID;
+ final private String containerMgrAddress;
+ private Token containerToken;
+
+ public Container(ContainerId containerID,
+ String containerMgrAddress, Token containerToken) {
+ this.state = ContainerState.PREP;
+ this.containerMgrAddress = containerMgrAddress;
+ this.containerID = containerID;
+ this.containerToken = containerToken;
+ }
+
+ public synchronized boolean isCompletelyDone() {
+ return state == ContainerState.DONE || state == ContainerState.FAILED;
+ }
+
+ @SuppressWarnings("unchecked")
+ public synchronized void launch(ContainerLaunchRequest event) {
+ LOG.info("Launching Container with Id: " + event.getContainerId());
+ if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
+ state = ContainerState.DONE;
+ sendContainerLaunchFailedMsg(event.getContainerId(),
+ "Container was killed before it was launched");
+ return;
+ }
+
+ ContainerManagementProtocolProxyData proxy = null;
+ try {
+
+ proxy = getCMProxy(containerID, containerMgrAddress,
+ containerToken);
+
+ // Construct the actual Container
+ ContainerLaunchContext containerLaunchContext =
+ event.getContainerLaunchContext();
+
+ // Now launch the actual container
+ StartContainerRequest startRequest = Records
+ .newRecord(StartContainerRequest.class);
+ startRequest.setContainerToken(event.getContainerToken());
+ startRequest.setContainerLaunchContext(containerLaunchContext);
+
+ StartContainersResponse response =
+ proxy.getContainerManagementProtocol().startContainers(
+ StartContainersRequest.newInstance(
+ Collections.singletonList(startRequest)));
+ if (response.getFailedRequests() != null
+ && !response.getFailedRequests().isEmpty()) {
+ throw response.getFailedRequests().get(containerID).deSerialize();
+ }
+
+ // after launching, send launched event to task attempt to move
+ // it from ASSIGNED to RUNNING state
+ getContext().containerLaunched(containerID);
+ this.state = ContainerState.RUNNING;
+ } catch (Throwable t) {
+ String message = "Container launch failed for " + containerID + " : "
+ + ExceptionUtils.getStackTrace(t);
+ this.state = ContainerState.FAILED;
+ sendContainerLaunchFailedMsg(containerID, message);
+ } finally {
+ if (proxy != null) {
+ cmProxy.mayBeCloseProxy(proxy);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public synchronized void kill() {
+
+ if(isCompletelyDone()) {
+ return;
+ }
+ if(this.state == ContainerState.PREP) {
+ this.state = ContainerState.KILLED_BEFORE_LAUNCH;
+ } else {
+ LOG.info("Sending a stop request to the NM for ContainerId: "
+ + containerID);
+
+ ContainerManagementProtocolProxyData proxy = null;
+ try {
+ proxy = getCMProxy(this.containerID, this.containerMgrAddress,
+ this.containerToken);
+
+ // kill the remote container if already launched
+ StopContainersRequest stopRequest = Records
+ .newRecord(StopContainersRequest.class);
+ stopRequest.setContainerIds(Collections.singletonList(containerID));
+
+ proxy.getContainerManagementProtocol().stopContainers(stopRequest);
+
+ // If stopContainer returns without an error, assuming the stop made
+ // it over to the NodeManager.
+ getContext().containerStopRequested(containerID);
+ } catch (Throwable t) {
+
+ // ignore the cleanup failure
+ String message = "cleanup failed for container "
+ + this.containerID + " : "
+ + ExceptionUtils.getStackTrace(t);
+ getContext().containerStopFailed(containerID, message);
+ LOG.warn(message);
+ this.state = ContainerState.DONE;
+ return;
+ } finally {
+ if (proxy != null) {
+ cmProxy.mayBeCloseProxy(proxy);
+ }
+ }
+ this.state = ContainerState.DONE;
+ }
+ }
+ }
+
+ public TezContainerLauncherImpl(ContainerLauncherContext containerLauncherContext) {
+ super(containerLauncherContext);
+ try {
+ this.conf = TezUtils.createConfFromUserPayload(containerLauncherContext.getInitialUserPayload());
+ } catch (IOException e) {
+ throw new TezUncheckedException(
+ "Failed to parse user payload for " + TezContainerLauncherImpl.class.getSimpleName(), e);
+ }
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
+ 0);
+ this.limitOnPoolSize = conf.getInt(
+ TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
+ TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT_DEFAULT);
+ LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize);
+ }
+
+ @Override
+ public void start() {
+ // pass a copy of config to ContainerManagementProtocolProxy until YARN-3497 is fixed
+ cmProxy =
+ new ContainerManagementProtocolProxy(conf);
+
+ ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
+ "ContainerLauncher #%d").setDaemon(true).build();
+
+ // Start with a default core-pool size of 10 and change it dynamically.
+ launcherPool = new ThreadPoolExecutor(INITIAL_POOL_SIZE,
+ Integer.MAX_VALUE, 1, TimeUnit.HOURS,
+ new LinkedBlockingQueue<Runnable>(),
+ tf, new CustomizedRejectedExecutionHandler());
+ eventHandlingThread = new Thread() {
+ @Override
+ public void run() {
+ ContainerOp event = null;
+ while (!Thread.currentThread().isInterrupted()) {
+ try {
+ event = eventQueue.take();
+ } catch (InterruptedException e) {
+ if(!serviceStopped.get()) {
+ LOG.error("Returning, interrupted : " + e);
+ }
+ return;
+ }
+ int poolSize = launcherPool.getCorePoolSize();
+
+ // See if we need up the pool size only if haven't reached the
+ // maximum limit yet.
+ if (poolSize != limitOnPoolSize) {
+
+ // nodes where containers will run at *this* point of time. This is
+ // *not* the cluster size and doesn't need to be.
+ int numNodes =
+ getContext().getNumNodes(TezConstants.getTezYarnServicePluginName());
+ int idealPoolSize = Math.min(limitOnPoolSize, numNodes);
+
+ if (poolSize < idealPoolSize) {
+ // Bump up the pool size to idealPoolSize+INITIAL_POOL_SIZE, the
+ // later is just a buffer so we are not always increasing the
+ // pool-size
+ int newPoolSize = Math.min(limitOnPoolSize, idealPoolSize
+ + INITIAL_POOL_SIZE);
+ LOG.info("Setting ContainerLauncher pool size to " + newPoolSize
+ + " as number-of-nodes to talk to is " + numNodes);
+ launcherPool.setCorePoolSize(newPoolSize);
+ }
+ }
+
+ // the events from the queue are handled in parallel
+ // using a thread pool
+ launcherPool.execute(createEventProcessor(event));
+
+ // TODO: Group launching of multiple containers to a single
+ // NodeManager into a single connection
+ }
+ }
+ };
+ eventHandlingThread.setName("ContainerLauncher Event Handler");
+ eventHandlingThread.start();
+ }
+
+ private void shutdownAllContainers() {
+ for (Container ct : this.containers.values()) {
+ if (ct != null) {
+ ct.kill();
+ }
+ }
+ }
+
+ @Override
+ public void shutdown() {
+ if(!serviceStopped.compareAndSet(false, true)) {
+ LOG.info("Ignoring multiple stops");
+ return;
+ }
+ // shutdown any containers that might be left running
+ shutdownAllContainers();
+ if (eventHandlingThread != null) {
+ eventHandlingThread.interrupt();
+ }
+ if (launcherPool != null) {
+ launcherPool.shutdownNow();
+ }
+ }
+
+ protected EventProcessor createEventProcessor(ContainerOp event) {
+ return new EventProcessor(event);
+ }
+
+ protected ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData getCMProxy(
+ ContainerId containerID, final String containerManagerBindAddr,
+ Token containerToken) throws IOException {
+ return cmProxy.getProxy(containerManagerBindAddr, containerID);
+ }
+
+ /**
+ * Setup and start the container on remote nodemanager.
+ */
+ class EventProcessor implements Runnable {
+ private ContainerOp event;
+
+ EventProcessor(ContainerOp event) {
+ this.event = event;
+ }
+
+ @Override
+ public void run() {
+ LOG.info("Processing operation {}", event.toString());
+
+ // Load ContainerManager tokens before creating a connection.
+ // TODO: Do it only once per NodeManager.
+ ContainerId containerID = event.getBaseOperation().getContainerId();
+
+ Container c = getContainer(event);
+ switch(event.getOpType()) {
+ case LAUNCH_REQUEST:
+ ContainerLaunchRequest launchRequest = event.getLaunchRequest();
+ c.launch(launchRequest);
+ break;
+ case STOP_REQUEST:
+ c.kill();
+ break;
+ }
+ removeContainerIfDone(containerID);
+ }
+ }
+
+ /**
+ * ThreadPoolExecutor.submit may fail if you are submitting task
+ * when ThreadPoolExecutor is shutting down (DAGAppMaster is shutting down).
+ * Use this CustomizedRejectedExecutionHandler to just logging rather than abort the application.
+ */
+ private static class CustomizedRejectedExecutionHandler implements RejectedExecutionHandler {
+ @Override
+ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
+ LOG.warn("Can't submit task to ThreadPoolExecutor:" + executor);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ void sendContainerLaunchFailedMsg(ContainerId containerId,
+ String message) {
+ LOG.error(message);
+ getContext().containerLaunchFailed(containerId, message);
+ }
+
+
+ @Override
+ public void launchContainer(ContainerLaunchRequest launchRequest) {
+ try {
+ eventQueue.put(new ContainerOp(ContainerOp.OPType.LAUNCH_REQUEST, launchRequest));
+ } catch (InterruptedException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+
+ @Override
+ public void stopContainer(ContainerStopRequest stopRequest) {
+ try {
+ eventQueue.put(new ContainerOp(ContainerOp.OPType.STOP_REQUEST, stopRequest));
+ } catch (InterruptedException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java
new file mode 100644
index 0000000..add3254
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+public class ContainerLauncherEvent extends AbstractEvent<ContainerLauncherEventType> {
+
+ private final ContainerId containerId;
+ private final NodeId nodeId;
+ private final Token containerToken;
+ private final int launcherId;
+ private final int schedulerId;
+ private final int taskCommId;
+
+ public ContainerLauncherEvent(ContainerId containerId, NodeId nodeId,
+ Token containerToken, ContainerLauncherEventType type,
+ int launcherId,
+ int schedulerId, int taskCommId) {
+ super(type);
+ this.containerId = containerId;
+ this.nodeId = nodeId;
+ this.containerToken = containerToken;
+ this.launcherId = launcherId;
+ this.schedulerId = schedulerId;
+ this.taskCommId = taskCommId;
+ }
+
+ public ContainerId getContainerId() {
+ return this.containerId;
+ }
+
+ public NodeId getNodeId() {
+ return this.nodeId;
+ }
+
+ public Token getContainerToken() {
+ return this.containerToken;
+ }
+
+ public int getLauncherId() {
+ return launcherId;
+ }
+
+ public int getSchedulerId() {
+ return schedulerId;
+ }
+
+ public int getTaskCommId() {
+ return taskCommId;
+ }
+
+ public String toSrting() {
+ return super.toString() + " for container " + containerId + ", nodeId: "
+ + nodeId + ", launcherId: " + launcherId + ", schedulerId=" + schedulerId +
+ ", taskCommId=" + taskCommId;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result
+ + ((containerId == null) ? 0 : containerId.hashCode());
+ result = prime * result
+ + ((containerToken == null) ? 0 : containerToken.hashCode());
+ result = prime * result + ((nodeId == null) ? 0 : nodeId.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ ContainerLauncherEvent other = (ContainerLauncherEvent) obj;
+ if (containerId == null) {
+ if (other.containerId != null)
+ return false;
+ } else if (!containerId.equals(other.containerId))
+ return false;
+ if (containerToken == null) {
+ if (other.containerToken != null)
+ return false;
+ } else if (!containerToken.equals(other.containerToken))
+ return false;
+ if (nodeId == null) {
+ if (other.nodeId != null)
+ return false;
+ } else if (!nodeId.equals(other.nodeId))
+ return false;
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java
new file mode 100644
index 0000000..079509c
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+// TODO - Re-use the events in ContainerLauncher..
+public enum ContainerLauncherEventType {
+ CONTAINER_LAUNCH_REQUEST,
+ CONTAINER_STOP_REQUEST
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java
new file mode 100644
index 0000000..411451e
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+
+public class ContainerLauncherLaunchRequestEvent extends ContainerLauncherEvent {
+
+ private final ContainerLaunchContext clc;
+ private final Container container;
+ // The task communicator index for the specific container being launched.
+
+ public ContainerLauncherLaunchRequestEvent(ContainerLaunchContext clc,
+ Container container, int launcherId, int schedulerId,
+ int taskCommId) {
+ super(container.getId(), container.getNodeId(), container
+ .getContainerToken(), ContainerLauncherEventType.CONTAINER_LAUNCH_REQUEST,
+ launcherId, schedulerId, taskCommId);
+ this.clc = clc;
+ this.container = container;
+ }
+
+ public ContainerLaunchContext getContainerLaunchContext() {
+ return this.clc;
+ }
+
+ public Container getContainer() {
+ return container;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+
+ ContainerLauncherLaunchRequestEvent that = (ContainerLauncherLaunchRequestEvent) o;
+
+ if (clc != null ? !clc.equals(that.clc) : that.clc != null) {
+ return false;
+ }
+ if (container != null ? !container.equals(that.container) : that.container != null) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = super.hashCode();
+ result = 7001 * result + (clc != null ? clc.hashCode() : 0);
+ result = 7001 * result + (container != null ? container.hashCode() : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java
new file mode 100644
index 0000000..69e7d30
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Token;
+
+public class ContainerLauncherStopRequestEvent extends ContainerLauncherEvent {
+
+ public ContainerLauncherStopRequestEvent(ContainerId containerId, NodeId nodeId,
+ Token containerToken, int launcherId, int schedulerId,
+ int taskCommId) {
+ super(containerId, nodeId, containerToken,
+ ContainerLauncherEventType.CONTAINER_STOP_REQUEST, launcherId, schedulerId, taskCommId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
deleted file mode 100644
index dc50c37..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-
-public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType> {
-
- private final ContainerId containerId;
- private final NodeId nodeId;
- private final Token containerToken;
- private final int launcherId;
- private final int schedulerId;
- private final int taskCommId;
-
- public NMCommunicatorEvent(ContainerId containerId, NodeId nodeId,
- Token containerToken, NMCommunicatorEventType type, int launcherId,
- int schedulerId, int taskCommId) {
- super(type);
- this.containerId = containerId;
- this.nodeId = nodeId;
- this.containerToken = containerToken;
- this.launcherId = launcherId;
- this.schedulerId = schedulerId;
- this.taskCommId = taskCommId;
- }
-
- public ContainerId getContainerId() {
- return this.containerId;
- }
-
- public NodeId getNodeId() {
- return this.nodeId;
- }
-
- public Token getContainerToken() {
- return this.containerToken;
- }
-
- public int getLauncherId() {
- return launcherId;
- }
-
- public int getSchedulerId() {
- return schedulerId;
- }
-
- public int getTaskCommId() {
- return taskCommId;
- }
-
- public String toSrting() {
- return super.toString() + " for container " + containerId + ", nodeId: "
- + nodeId + ", launcherId: " + launcherId + ", schedulerId=" + schedulerId +
- ", taskCommId=" + taskCommId;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result
- + ((containerId == null) ? 0 : containerId.hashCode());
- result = prime * result
- + ((containerToken == null) ? 0 : containerToken.hashCode());
- result = prime * result + ((nodeId == null) ? 0 : nodeId.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- NMCommunicatorEvent other = (NMCommunicatorEvent) obj;
- if (containerId == null) {
- if (other.containerId != null)
- return false;
- } else if (!containerId.equals(other.containerId))
- return false;
- if (containerToken == null) {
- if (other.containerToken != null)
- return false;
- } else if (!containerToken.equals(other.containerToken))
- return false;
- if (nodeId == null) {
- if (other.nodeId != null)
- return false;
- } else if (!nodeId.equals(other.nodeId))
- return false;
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEventType.java
deleted file mode 100644
index 9f3d989..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEventType.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-// TODO - Re-use the events in ContainerLauncher..
-public enum NMCommunicatorEventType {
- CONTAINER_LAUNCH_REQUEST,
- CONTAINER_STOP_REQUEST
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java
deleted file mode 100644
index c57b6be..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorLaunchRequestEvent.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-
-public class NMCommunicatorLaunchRequestEvent extends NMCommunicatorEvent {
-
- private final ContainerLaunchContext clc;
- private final Container container;
- // The task communicator index for the specific container being launched.
-
- public NMCommunicatorLaunchRequestEvent(ContainerLaunchContext clc,
- Container container, int launcherId, int schedulerId, int taskCommId) {
- super(container.getId(), container.getNodeId(), container
- .getContainerToken(), NMCommunicatorEventType.CONTAINER_LAUNCH_REQUEST,
- launcherId, schedulerId, taskCommId);
- this.clc = clc;
- this.container = container;
- }
-
- public ContainerLaunchContext getContainerLaunchContext() {
- return this.clc;
- }
-
- public Container getContainer() {
- return container;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
-
- NMCommunicatorLaunchRequestEvent that = (NMCommunicatorLaunchRequestEvent) o;
-
- if (clc != null ? !clc.equals(that.clc) : that.clc != null) {
- return false;
- }
- if (container != null ? !container.equals(that.container) : that.container != null) {
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = super.hashCode();
- result = 7001 * result + (clc != null ? clc.hashCode() : 0);
- result = 7001 * result + (container != null ? container.hashCode() : 0);
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
deleted file mode 100644
index 352f450..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Token;
-
-public class NMCommunicatorStopRequestEvent extends NMCommunicatorEvent {
-
- public NMCommunicatorStopRequestEvent(ContainerId containerId, NodeId nodeId,
- Token containerToken, int launcherId, int schedulerId, int taskCommId) {
- super(containerId, nodeId, containerToken,
- NMCommunicatorEventType.CONTAINER_STOP_REQUEST, launcherId, schedulerId, taskCommId);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java
index 2a9797f..37aa96b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java
@@ -33,7 +33,7 @@ import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
public class TaskSchedulerContextImpl implements TaskSchedulerContext {
- private final TaskSchedulerEventHandler tseh;
+ private final TaskSchedulerManager taskSchedulerManager;
private final AppContext appContext;
private final int schedulerId;
private final String trackingUrl;
@@ -42,11 +42,11 @@ public class TaskSchedulerContextImpl implements TaskSchedulerContext {
private final int clientPort;
private final UserPayload initialUserPayload;
- public TaskSchedulerContextImpl(TaskSchedulerEventHandler tseh, AppContext appContext,
+ public TaskSchedulerContextImpl(TaskSchedulerManager taskSchedulerManager, AppContext appContext,
int schedulerId, String trackingUrl, long customClusterIdentifier,
String appHostname, int clientPort,
UserPayload initialUserPayload) {
- this.tseh = tseh;
+ this.taskSchedulerManager = taskSchedulerManager;
this.appContext = appContext;
this.schedulerId = schedulerId;
this.trackingUrl = trackingUrl;
@@ -62,54 +62,55 @@ public class TaskSchedulerContextImpl implements TaskSchedulerContext {
// taskAllocated() upcall and deallocateTask() downcall
@Override
public void taskAllocated(Object task, Object appCookie, Container container) {
- tseh.taskAllocated(schedulerId, task, appCookie, container);
+ taskSchedulerManager.taskAllocated(schedulerId, task, appCookie, container);
}
@Override
public void containerCompleted(Object taskLastAllocated, ContainerStatus containerStatus) {
- tseh.containerCompleted(schedulerId, taskLastAllocated, containerStatus);
+ taskSchedulerManager.containerCompleted(schedulerId, taskLastAllocated, containerStatus);
}
@Override
public void containerBeingReleased(ContainerId containerId) {
- tseh.containerBeingReleased(schedulerId, containerId);
+ taskSchedulerManager.containerBeingReleased(schedulerId, containerId);
}
@Override
public void nodesUpdated(List<NodeReport> updatedNodes) {
- tseh.nodesUpdated(schedulerId, updatedNodes);
+ taskSchedulerManager.nodesUpdated(schedulerId, updatedNodes);
}
@Override
public void appShutdownRequested() {
- tseh.appShutdownRequested(schedulerId);
+ taskSchedulerManager.appShutdownRequested(schedulerId);
}
@Override
public void setApplicationRegistrationData(Resource maxContainerCapability,
Map<ApplicationAccessType, String> appAcls,
ByteBuffer clientAMSecretKey) {
- tseh.setApplicationRegistrationData(schedulerId, maxContainerCapability, appAcls, clientAMSecretKey);
+ taskSchedulerManager.setApplicationRegistrationData(schedulerId, maxContainerCapability, appAcls,
+ clientAMSecretKey);
}
@Override
public void onError(Throwable t) {
- tseh.onError(schedulerId, t);
+ taskSchedulerManager.onError(schedulerId, t);
}
@Override
public float getProgress() {
- return tseh.getProgress(schedulerId);
+ return taskSchedulerManager.getProgress(schedulerId);
}
@Override
public void preemptContainer(ContainerId containerId) {
- tseh.preemptContainer(schedulerId, containerId);
+ taskSchedulerManager.preemptContainer(schedulerId, containerId);
}
@Override
public AppFinalStatus getFinalAppStatus() {
- return tseh.getFinalAppStatus();
+ return taskSchedulerManager.getFinalAppStatus();
}
@Override
@@ -130,7 +131,7 @@ public class TaskSchedulerContextImpl implements TaskSchedulerContext {
@Override
public ContainerSignatureMatcher getContainerSignatureMatcher() {
- return tseh.getContainerSignatureMatcher();
+ return taskSchedulerManager.getContainerSignatureMatcher();
}
@Override
[5/7] tez git commit: TEZ-2708. Rename classes and variables post
TEZ-2003 changes. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
deleted file mode 100644
index 76d870c..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ /dev/null
@@ -1,784 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
-import org.apache.tez.serviceplugins.api.TaskScheduler;
-import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
-import org.apache.tez.serviceplugins.api.TaskSchedulerContext.AppFinalStatus;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.common.ReflectionUtils;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.TaskLocationHint;
-import org.apache.tez.dag.api.TaskLocationHint.TaskBasedLocationAffinity;
-import org.apache.tez.dag.api.client.DAGClientServer;
-import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
-import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.DAGAppMaster;
-import org.apache.tez.dag.app.DAGAppMasterState;
-import org.apache.tez.dag.app.dag.TaskAttempt;
-import org.apache.tez.dag.app.dag.Vertex;
-import org.apache.tez.dag.app.dag.event.DAGAppMasterEvent;
-import org.apache.tez.dag.app.dag.event.DAGAppMasterEventSchedulingServiceError;
-import org.apache.tez.dag.app.dag.event.DAGAppMasterEventType;
-import org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdateTAAssigned;
-import org.apache.tez.dag.app.rm.container.AMContainer;
-import org.apache.tez.dag.app.rm.container.AMContainerEventAssignTA;
-import org.apache.tez.dag.app.rm.container.AMContainerEventCompleted;
-import org.apache.tez.dag.app.rm.container.AMContainerEventLaunchRequest;
-import org.apache.tez.dag.app.rm.container.AMContainerEventStopRequest;
-import org.apache.tez.dag.app.rm.container.AMContainerEventTASucceeded;
-import org.apache.tez.dag.app.rm.container.AMContainerState;
-import org.apache.tez.common.ContainerSignatureMatcher;
-import org.apache.tez.dag.app.rm.node.AMNodeEventContainerAllocated;
-import org.apache.tez.dag.app.rm.node.AMNodeEventNodeCountUpdated;
-import org.apache.tez.dag.app.rm.node.AMNodeEventStateChanged;
-import org.apache.tez.dag.app.rm.node.AMNodeEventTaskAttemptEnded;
-import org.apache.tez.dag.app.rm.node.AMNodeEventTaskAttemptSucceeded;
-import org.apache.tez.dag.app.web.WebUIService;
-import org.apache.tez.dag.records.TaskAttemptTerminationCause;
-
-import com.google.common.base.Preconditions;
-
-
-public class TaskSchedulerEventHandler extends AbstractService implements
- EventHandler<AMSchedulerEvent> {
- static final Logger LOG = LoggerFactory.getLogger(TaskSchedulerEventHandler.class);
-
- static final String APPLICATION_ID_PLACEHOLDER = "__APPLICATION_ID__";
- static final String HISTORY_URL_BASE = "__HISTORY_URL_BASE__";
-
- protected final AppContext appContext;
- @SuppressWarnings("rawtypes")
- private final EventHandler eventHandler;
- private final String historyUrl;
- private DAGAppMaster dagAppMaster;
- private Map<ApplicationAccessType, String> appAcls = null;
- private Thread eventHandlingThread;
- private volatile boolean stopEventHandling;
- // Has a signal (SIGTERM etc) been issued?
- protected volatile boolean isSignalled = false;
- final DAGClientServer clientService;
- private final ContainerSignatureMatcher containerSignatureMatcher;
- private int cachedNodeCount = -1;
- private AtomicBoolean shouldUnregisterFlag =
- new AtomicBoolean(false);
- private final WebUIService webUI;
- private final NamedEntityDescriptor[] taskSchedulerDescriptors;
- protected final TaskScheduler[]taskSchedulers;
- protected final ServicePluginLifecycleAbstractService []taskSchedulerServiceWrappers;
-
- // Single executor service shared by all Schedulers for context callbacks
- @VisibleForTesting
- final ExecutorService appCallbackExecutor;
-
- private final boolean isPureLocalMode;
- // If running in non local-only mode, the YARN task scheduler will always run to take care of
- // registration with YARN and heartbeats to YARN.
- // Splitting registration and heartbeats is not straight-forward due to the taskScheduler being
- // tied to a ContainerRequestType.
- // Custom AppIds to avoid container conflicts if there's multiple sources
- private final long SCHEDULER_APP_ID_BASE = 111101111;
- private final long SCHEDULER_APP_ID_INCREMENT = 111111111;
-
- BlockingQueue<AMSchedulerEvent> eventQueue
- = new LinkedBlockingQueue<AMSchedulerEvent>();
-
- // Not tracking container / task to schedulerId. Instead relying on everything flowing through
- // the system and being propagated back via events.
-
- /**
- *
- * @param appContext
- * @param clientService
- * @param eventHandler
- * @param containerSignatureMatcher
- * @param webUI
- * @param schedulerDescriptors the list of scheduler descriptors. Tez internal classes will not have the class names populated.
- * An empty list defaults to using the YarnTaskScheduler as the only source.
- * @param isPureLocalMode whether the AM is running in local mode
- */
- @SuppressWarnings("rawtypes")
- public TaskSchedulerEventHandler(AppContext appContext,
- DAGClientServer clientService, EventHandler eventHandler,
- ContainerSignatureMatcher containerSignatureMatcher, WebUIService webUI,
- List<NamedEntityDescriptor> schedulerDescriptors, boolean isPureLocalMode) {
- super(TaskSchedulerEventHandler.class.getName());
- Preconditions.checkArgument(schedulerDescriptors != null && !schedulerDescriptors.isEmpty(),
- "TaskSchedulerDescriptors must be specified");
- this.appContext = appContext;
- this.eventHandler = eventHandler;
- this.clientService = clientService;
- this.containerSignatureMatcher = containerSignatureMatcher;
- this.webUI = webUI;
- this.historyUrl = getHistoryUrl();
- this.isPureLocalMode = isPureLocalMode;
- this.appCallbackExecutor = createAppCallbackExecutorService();
- if (this.webUI != null) {
- this.webUI.setHistoryUrl(this.historyUrl);
- }
-
- this.taskSchedulerDescriptors = schedulerDescriptors.toArray(new NamedEntityDescriptor[schedulerDescriptors.size()]);
-
- taskSchedulers = new TaskScheduler[this.taskSchedulerDescriptors.length];
- taskSchedulerServiceWrappers = new ServicePluginLifecycleAbstractService[this.taskSchedulerDescriptors.length];
- }
-
- public Map<ApplicationAccessType, String> getApplicationAcls() {
- return appAcls;
- }
-
- public void setSignalled(boolean isSignalled) {
- this.isSignalled = isSignalled;
- LOG.info("TaskScheduler notified that iSignalled was : " + isSignalled);
- }
-
- public int getNumClusterNodes() {
- return cachedNodeCount;
- }
-
- public Resource getAvailableResources(int schedulerId) {
- return taskSchedulers[schedulerId].getAvailableResources();
- }
-
- public Resource getTotalResources(int schedulerId) {
- return taskSchedulers[schedulerId].getTotalResources();
- }
-
- private ExecutorService createAppCallbackExecutorService() {
- return Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("TaskSchedulerAppCallbackExecutor #%d")
- .setDaemon(true)
- .build());
- }
-
- public synchronized void handleEvent(AMSchedulerEvent sEvent) {
- LOG.info("Processing the event " + sEvent.toString());
- switch (sEvent.getType()) {
- case S_TA_LAUNCH_REQUEST:
- handleTaLaunchRequest((AMSchedulerEventTALaunchRequest) sEvent);
- break;
- case S_TA_ENDED: // TaskAttempt considered complete.
- AMSchedulerEventTAEnded event = (AMSchedulerEventTAEnded)sEvent;
- switch(event.getState()) {
- case FAILED:
- case KILLED:
- handleTAUnsuccessfulEnd(event);
- break;
- case SUCCEEDED:
- handleTASucceeded(event);
- break;
- default:
- throw new TezUncheckedException("Unexecpted TA_ENDED state: " + event.getState());
- }
- break;
- case S_CONTAINER_DEALLOCATE:
- handleContainerDeallocate((AMSchedulerEventDeallocateContainer)sEvent);
- break;
- case S_NODE_UNBLACKLISTED:
- // fall through
- case S_NODE_BLACKLISTED:
- handleNodeBlacklistUpdate((AMSchedulerEventNodeBlacklistUpdate)sEvent);
- break;
- case S_NODE_UNHEALTHY:
- break;
- case S_NODE_HEALTHY:
- // Consider changing this to work like BLACKLISTING.
- break;
- default:
- break;
- }
- }
-
- @Override
- public void handle(AMSchedulerEvent event) {
- int qSize = eventQueue.size();
- if (qSize != 0 && qSize % 1000 == 0) {
- LOG.info("Size of event-queue in RMContainerAllocator is " + qSize);
- }
- int remCapacity = eventQueue.remainingCapacity();
- if (remCapacity < 1000) {
- LOG.warn("Very low remaining capacity in the event-queue "
- + "of RMContainerAllocator: " + remCapacity);
- }
- try {
- eventQueue.put(event);
- } catch (InterruptedException e) {
- throw new TezUncheckedException(e);
- }
- }
-
- @SuppressWarnings("unchecked")
- private void sendEvent(Event<?> event) {
- eventHandler.handle(event);
- }
-
- private void handleNodeBlacklistUpdate(AMSchedulerEventNodeBlacklistUpdate event) {
- if (event.getType() == AMSchedulerEventType.S_NODE_BLACKLISTED) {
- taskSchedulers[event.getSchedulerId()].blacklistNode(event.getNodeId());
- } else if (event.getType() == AMSchedulerEventType.S_NODE_UNBLACKLISTED) {
- taskSchedulers[event.getSchedulerId()].unblacklistNode(event.getNodeId());
- } else {
- throw new TezUncheckedException("Invalid event type: " + event.getType());
- }
- }
-
- private void handleContainerDeallocate(
- AMSchedulerEventDeallocateContainer event) {
- ContainerId containerId = event.getContainerId();
- // TODO what happens to the task that was connected to this container?
- // current assumption is that it will eventually call handleTaStopRequest
- //TaskAttempt taskAttempt = (TaskAttempt)
- taskSchedulers[event.getSchedulerId()].deallocateContainer(containerId);
- // TODO does this container need to be stopped via C_STOP_REQUEST
- sendEvent(new AMContainerEventStopRequest(containerId));
- }
-
- private void handleTAUnsuccessfulEnd(AMSchedulerEventTAEnded event) {
- TaskAttempt attempt = event.getAttempt();
- // Propagate state and failure cause (if any) when informing the scheduler about the de-allocation.
- boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()]
- .deallocateTask(attempt, false, event.getTaskAttemptEndReason(), event.getDiagnostics());
- // use stored value of container id in case the scheduler has removed this
- // assignment because the task has been deallocated earlier.
- // retroactive case
- ContainerId attemptContainerId = attempt.getAssignedContainerID();
-
- if(!wasContainerAllocated) {
- LOG.info("Task: " + attempt.getID() +
- " has no container assignment in the scheduler");
- if (attemptContainerId != null) {
- LOG.error("No container allocated to task: " + attempt.getID()
- + " according to scheduler. Task reported container id: "
- + attemptContainerId);
- }
- }
-
- if (attemptContainerId != null) {
- // TODO either ways send the necessary events
- // Ask the container to stop.
- sendEvent(new AMContainerEventStopRequest(attemptContainerId));
- // Inform the Node - the task has asked to be STOPPED / has already
- // stopped.
- // AMNodeImpl blacklisting logic does not account for KILLED attempts.
- sendEvent(new AMNodeEventTaskAttemptEnded(appContext.getAllContainers().
- get(attemptContainerId).getContainer().getNodeId(), event.getSchedulerId(),
- attemptContainerId,
- attempt.getID(), event.getState() == TaskAttemptState.FAILED));
- }
- }
-
- private void handleTASucceeded(AMSchedulerEventTAEnded event) {
- TaskAttempt attempt = event.getAttempt();
- ContainerId usedContainerId = event.getUsedContainerId();
-
- // This could be null if a task fails / is killed before a container is
- // assigned to it.
- if (event.getUsedContainerId() != null) {
- sendEvent(new AMContainerEventTASucceeded(usedContainerId,
- event.getAttemptID()));
- sendEvent(new AMNodeEventTaskAttemptSucceeded(appContext.getAllContainers().
- get(usedContainerId).getContainer().getNodeId(), event.getSchedulerId(), usedContainerId,
- event.getAttemptID()));
- }
-
- boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()].deallocateTask(attempt,
- true, null, event.getDiagnostics());
- if (!wasContainerAllocated) {
- LOG.error("De-allocated successful task: " + attempt.getID()
- + ", but TaskScheduler reported no container assigned to task");
- }
- }
-
- private void handleTaLaunchRequest(AMSchedulerEventTALaunchRequest event) {
- TaskAttempt taskAttempt = event.getTaskAttempt();
- TaskLocationHint locationHint = event.getLocationHint();
- String hosts[] = null;
- String racks[] = null;
- if (locationHint != null) {
- TaskBasedLocationAffinity taskAffinity = locationHint.getAffinitizedTask();
- if (taskAffinity != null) {
- Vertex vertex = appContext.getCurrentDAG().getVertex(taskAffinity.getVertexName());
- Preconditions.checkNotNull(vertex, "Invalid vertex in task based affinity " + taskAffinity
- + " for attempt: " + taskAttempt.getID());
- int taskIndex = taskAffinity.getTaskIndex();
- Preconditions.checkState(taskIndex >=0 && taskIndex < vertex.getTotalTasks(),
- "Invalid taskIndex in task based affinity " + taskAffinity
- + " for attempt: " + taskAttempt.getID());
- TaskAttempt affinityAttempt = vertex.getTask(taskIndex).getSuccessfulAttempt();
- if (affinityAttempt != null) {
- Preconditions.checkNotNull(affinityAttempt.getAssignedContainerID(), affinityAttempt.getID());
- taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt,
- event.getCapability(),
- affinityAttempt.getAssignedContainerID(),
- Priority.newInstance(event.getPriority()),
- event.getContainerContext(),
- event);
- return;
- }
- LOG.info("Attempt: " + taskAttempt.getID() + " has task based affinity to " + taskAffinity
- + " but no locality information exists for it. Ignoring hint.");
- // fall through with null hosts/racks
- } else {
- hosts = (locationHint.getHosts() != null) ? locationHint
- .getHosts().toArray(
- new String[locationHint.getHosts().size()]) : null;
- racks = (locationHint.getRacks() != null) ? locationHint.getRacks()
- .toArray(new String[locationHint.getRacks().size()]) : null;
- }
- }
-
- taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt,
- event.getCapability(),
- hosts,
- racks,
- Priority.newInstance(event.getPriority()),
- event.getContainerContext(),
- event);
- }
-
- @VisibleForTesting
- TaskScheduler createTaskScheduler(String host, int port, String trackingUrl,
- AppContext appContext,
- NamedEntityDescriptor taskSchedulerDescriptor,
- long customAppIdIdentifier,
- int schedulerId) {
- TaskSchedulerContext rawContext =
- new TaskSchedulerContextImpl(this, appContext, schedulerId, trackingUrl,
- customAppIdIdentifier, host, port, taskSchedulerDescriptor.getUserPayload());
- TaskSchedulerContext wrappedContext = wrapTaskSchedulerContext(rawContext);
- String schedulerName = taskSchedulerDescriptor.getEntityName();
- if (schedulerName.equals(TezConstants.getTezYarnServicePluginName())) {
- return createYarnTaskScheduler(wrappedContext, schedulerId);
- } else if (schedulerName.equals(TezConstants.getTezUberServicePluginName())) {
- return createUberTaskScheduler(wrappedContext, schedulerId);
- } else {
- return createCustomTaskScheduler(wrappedContext, taskSchedulerDescriptor, schedulerId);
- }
- }
-
- @VisibleForTesting
- TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
- return new TaskSchedulerContextImplWrapper(rawContext, appCallbackExecutor);
- }
-
- @VisibleForTesting
- TaskScheduler createYarnTaskScheduler(TaskSchedulerContext taskSchedulerContext,
- int schedulerId) {
- LOG.info("Creating TaskScheduler: YarnTaskSchedulerService");
- return new YarnTaskSchedulerService(taskSchedulerContext);
- }
-
- @VisibleForTesting
- TaskScheduler createUberTaskScheduler(TaskSchedulerContext taskSchedulerContext,
- int schedulerId) {
- LOG.info("Creating TaskScheduler: Local TaskScheduler");
- return new LocalTaskSchedulerService(taskSchedulerContext);
- }
-
- @SuppressWarnings("unchecked")
- TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext,
- NamedEntityDescriptor taskSchedulerDescriptor,
- int schedulerId) {
- LOG.info("Creating custom TaskScheduler {}:{}", taskSchedulerDescriptor.getEntityName(),
- taskSchedulerDescriptor.getClassName());
- Class<? extends TaskScheduler> taskSchedulerClazz =
- (Class<? extends TaskScheduler>) ReflectionUtils
- .getClazz(taskSchedulerDescriptor.getClassName());
- try {
- Constructor<? extends TaskScheduler> ctor = taskSchedulerClazz
- .getConstructor(TaskSchedulerContext.class);
- return ctor.newInstance(taskSchedulerContext);
- } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
- throw new TezUncheckedException(e);
- }
- }
-
- @VisibleForTesting
- protected void instantiateSchedulers(String host, int port, String trackingUrl,
- AppContext appContext) {
- // Iterate over the list and create all the taskSchedulers
- int j = 0;
- for (int i = 0; i < taskSchedulerDescriptors.length; i++) {
- long customAppIdIdentifier;
- if (isPureLocalMode || taskSchedulerDescriptors[i].getEntityName().equals(
- TezConstants.getTezYarnServicePluginName())) { // Use the app identifier from the appId.
- customAppIdIdentifier = appContext.getApplicationID().getClusterTimestamp();
- } else {
- customAppIdIdentifier = SCHEDULER_APP_ID_BASE + (j++ * SCHEDULER_APP_ID_INCREMENT);
- }
- LOG.info("ClusterIdentifier for TaskScheduler [" + i + ":" + taskSchedulerDescriptors[i].getEntityName() + "]=" +
- customAppIdIdentifier);
- taskSchedulers[i] = createTaskScheduler(host, port,
- trackingUrl, appContext, taskSchedulerDescriptors[i], customAppIdIdentifier, i);
- taskSchedulerServiceWrappers[i] = new ServicePluginLifecycleAbstractService<>(taskSchedulers[i]);
- }
- }
-
-
- @Override
- public synchronized void serviceStart() {
- InetSocketAddress serviceAddr = clientService.getBindAddress();
- dagAppMaster = appContext.getAppMaster();
- // if web service is enabled then set tracking url. else disable it (value = "").
- // the actual url set on the rm web ui will be the proxy url set by WebAppProxyServlet, which
- // always try to connect to AM and proxy the response. hence it wont work if the webUIService
- // is not enabled.
- String trackingUrl = (webUI != null) ? webUI.getTrackingURL() : "";
- instantiateSchedulers(serviceAddr.getHostName(), serviceAddr.getPort(), trackingUrl, appContext);
-
- for (int i = 0 ; i < taskSchedulers.length ; i++) {
- taskSchedulerServiceWrappers[i].init(getConfig());
- taskSchedulerServiceWrappers[i].start();
- if (shouldUnregisterFlag.get()) {
- // Flag may have been set earlier when task scheduler was not initialized
- // External services could need to talk to some other entity.
- taskSchedulers[i].setShouldUnregister();
- }
- }
-
- this.eventHandlingThread = new Thread("TaskSchedulerEventHandlerThread") {
- @Override
- public void run() {
-
- AMSchedulerEvent event;
-
- while (!stopEventHandling && !Thread.currentThread().isInterrupted()) {
- try {
- if (TaskSchedulerEventHandler.this.eventQueue.peek() == null) {
- notifyForTest();
- }
- event = TaskSchedulerEventHandler.this.eventQueue.take();
- } catch (InterruptedException e) {
- if(!stopEventHandling) {
- LOG.warn("Continuing after interrupt : ", e);
- }
- continue;
- }
-
- try {
- handleEvent(event);
- } catch (Throwable t) {
- LOG.error("Error in handling event type " + event.getType()
- + " to the TaskScheduler", t);
- // Kill the AM.
- sendEvent(new DAGAppMasterEvent(DAGAppMasterEventType.INTERNAL_ERROR));
- return;
- } finally {
- notifyForTest();
- }
- }
- }
- };
- this.eventHandlingThread.start();
- }
-
- protected void notifyForTest() {
- }
-
- public void initiateStop() {
- for (int i = 0 ; i < taskSchedulers.length ; i++) {
- taskSchedulers[i].initiateStop();
- }
- }
-
- @Override
- public void serviceStop() throws InterruptedException {
- synchronized(this) {
- this.stopEventHandling = true;
- if (eventHandlingThread != null)
- eventHandlingThread.interrupt();
- }
- for (int i = 0 ; i < taskSchedulers.length ; i++) {
- if (taskSchedulers[i] != null) {
- taskSchedulerServiceWrappers[i].stop();
- }
- }
- LOG.info("Shutting down AppCallbackExecutor");
- appCallbackExecutor.shutdownNow();
- appCallbackExecutor.awaitTermination(1000l, TimeUnit.MILLISECONDS);
- }
-
- // TaskSchedulerAppCallback methods with schedulerId, where relevant
- public synchronized void taskAllocated(int schedulerId, Object task,
- Object appCookie,
- Container container) {
- AMSchedulerEventTALaunchRequest event =
- (AMSchedulerEventTALaunchRequest) appCookie;
- ContainerId containerId = container.getId();
- if (appContext.getAllContainers()
- .addContainerIfNew(container, schedulerId, event.getLauncherId(),
- event.getTaskCommId())) {
- appContext.getNodeTracker().nodeSeen(container.getNodeId(), schedulerId);
- sendEvent(new AMNodeEventContainerAllocated(container
- .getNodeId(), schedulerId, container.getId()));
- }
-
-
- TaskAttempt taskAttempt = event.getTaskAttempt();
- // TODO - perhaps check if the task still needs this container
- // because the deallocateTask downcall may have raced with the
- // taskAllocated() upcall
- assert task.equals(taskAttempt);
-
- if (appContext.getAllContainers().get(containerId).getState() == AMContainerState.ALLOCATED) {
- sendEvent(new AMContainerEventLaunchRequest(containerId, taskAttempt.getVertexID(),
- event.getContainerContext(), event.getLauncherId(), event.getTaskCommId()));
- }
- sendEvent(new DAGEventSchedulerUpdateTAAssigned(taskAttempt, container));
- sendEvent(new AMContainerEventAssignTA(containerId, taskAttempt.getID(),
- event.getRemoteTaskSpec(), event.getContainerContext().getLocalResources(), event
- .getContainerContext().getCredentials(), event.getPriority()));
- }
-
- public synchronized void containerCompleted(int schedulerId, Object task, ContainerStatus containerStatus) {
- // SchedulerId isn't used here since no node updates are sent out
- // Inform the Containers about completion.
- AMContainer amContainer = appContext.getAllContainers().get(containerStatus.getContainerId());
- if (amContainer != null) {
- String message = "Container completed. ";
- TaskAttemptTerminationCause errCause = TaskAttemptTerminationCause.CONTAINER_EXITED;
- int exitStatus = containerStatus.getExitStatus();
- if (exitStatus == ContainerExitStatus.PREEMPTED) {
- message = "Container preempted externally. ";
- errCause = TaskAttemptTerminationCause.EXTERNAL_PREEMPTION;
- } else if (exitStatus == ContainerExitStatus.DISKS_FAILED) {
- message = "Container disk failed. ";
- errCause = TaskAttemptTerminationCause.NODE_DISK_ERROR;
- } else if (exitStatus != ContainerExitStatus.SUCCESS){
- message = "Container failed, exitCode=" + exitStatus + ". ";
- }
- if (containerStatus.getDiagnostics() != null) {
- message += containerStatus.getDiagnostics();
- }
- sendEvent(new AMContainerEventCompleted(amContainer.getContainerId(), exitStatus, message, errCause));
- }
- }
-
- public synchronized void containerBeingReleased(int schedulerId, ContainerId containerId) {
- // SchedulerId isn't used here since no node updates are sent out
- AMContainer amContainer = appContext.getAllContainers().get(containerId);
- if (amContainer != null) {
- sendEvent(new AMContainerEventStopRequest(containerId));
- }
- }
-
- @SuppressWarnings("unchecked")
- public synchronized void nodesUpdated(int schedulerId, List<NodeReport> updatedNodes) {
- for (NodeReport nr : updatedNodes) {
- // Scheduler will find out from the node, if at all.
- // Relying on the RM to not allocate containers on an unhealthy node.
- eventHandler.handle(new AMNodeEventStateChanged(nr, schedulerId));
- }
- }
-
- public synchronized void appShutdownRequested(int schedulerId) {
- // This can happen if the RM has been restarted. If it is in that state,
- // this application must clean itself up.
- LOG.info("App shutdown requested by scheduler {}", schedulerId);
- sendEvent(new DAGAppMasterEvent(DAGAppMasterEventType.AM_REBOOT));
- }
-
- public synchronized void setApplicationRegistrationData(
- int schedulerId,
- Resource maxContainerCapability,
- Map<ApplicationAccessType, String> appAcls,
- ByteBuffer clientAMSecretKey) {
- this.appContext.getClusterInfo().setMaxContainerCapability(
- maxContainerCapability);
- this.appAcls = appAcls;
- this.clientService.setClientAMSecretKey(clientAMSecretKey);
- }
-
- // Not synchronized to avoid deadlocks from TaskScheduler callbacks.
- // TaskScheduler uses a separate thread for it's callbacks. Since this method
- // returns a value which is required, the TaskScheduler wait for the call to
- // complete and can hence lead to a deadlock if called from within a TSEH lock.
- public AppFinalStatus getFinalAppStatus() {
- FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
- StringBuffer sb = new StringBuffer();
- if (dagAppMaster == null) {
- finishState = FinalApplicationStatus.UNDEFINED;
- sb.append("App not yet initialized");
- } else {
- DAGAppMasterState appMasterState = dagAppMaster.getState();
- if (appMasterState == DAGAppMasterState.SUCCEEDED) {
- finishState = FinalApplicationStatus.SUCCEEDED;
- } else if (appMasterState == DAGAppMasterState.KILLED
- || (appMasterState == DAGAppMasterState.RUNNING && isSignalled)) {
- finishState = FinalApplicationStatus.KILLED;
- } else if (appMasterState == DAGAppMasterState.FAILED
- || appMasterState == DAGAppMasterState.ERROR) {
- finishState = FinalApplicationStatus.FAILED;
- } else {
- finishState = FinalApplicationStatus.UNDEFINED;
- }
- List<String> diagnostics = dagAppMaster.getDiagnostics();
- if(diagnostics != null) {
- for (String s : diagnostics) {
- sb.append(s).append("\n");
- }
- }
- }
- if(LOG.isDebugEnabled()) {
- LOG.debug("Setting job diagnostics to " + sb.toString());
- }
-
- // if history url is set use the same, if historyUrl is set to "" then rm ui disables the
- // history url
- return new AppFinalStatus(finishState, sb.toString(), historyUrl);
- }
-
-
-
- // Not synchronized to avoid deadlocks from TaskScheduler callbacks.
- // TaskScheduler uses a separate thread for it's callbacks. Since this method
- // returns a value which is required, the TaskScheduler wait for the call to
- // complete and can hence lead to a deadlock if called from within a TSEH lock.
- public float getProgress(int schedulerId) {
- // at this point allocate has been called and so node count must be available
- // may change after YARN-1722
- // This is a heartbeat in from the scheduler into the APP, and is being used to piggy-back and
- // node updates from the cluster.
-
- // Doubles as a mechanism to update node counts periodically. Hence schedulerId required.
-
- // TODO Handle this in TEZ-2124. Need a way to know which scheduler is calling in.
- int nodeCount = taskSchedulers[0].getClusterNodeCount();
- if (nodeCount != cachedNodeCount) {
- cachedNodeCount = nodeCount;
- sendEvent(new AMNodeEventNodeCountUpdated(cachedNodeCount, schedulerId));
- }
- return dagAppMaster.getProgress();
- }
-
- public void onError(int schedulerId, Throwable t) {
- LOG.info("Error reported by scheduler {} - {}", schedulerId, t);
- sendEvent(new DAGAppMasterEventSchedulingServiceError(t));
- }
-
- public void dagCompleted() {
- for (int i = 0 ; i < taskSchedulers.length ; i++) {
- taskSchedulers[i].dagComplete();
- }
- }
-
- public void dagSubmitted() {
- // Nothing to do right now. Indicates that a new DAG has been submitted and
- // the context has updated information.
- }
-
- public void preemptContainer(int schedulerId, ContainerId containerId) {
- // TODO Why is this making a call back into the scheduler, when the call is originating from there.
- // An AMContainer instance should already exist if an attempt is being made to preempt it
- AMContainer amContainer = appContext.getAllContainers().get(containerId);
- taskSchedulers[amContainer.getTaskSchedulerIdentifier()].deallocateContainer(containerId);
- // Inform the Containers about completion.
- sendEvent(new AMContainerEventCompleted(containerId, ContainerExitStatus.INVALID,
- "Container preempted internally", TaskAttemptTerminationCause.INTERNAL_PREEMPTION));
- }
-
- public void setShouldUnregisterFlag() {
- LOG.info("TaskScheduler notified that it should unregister from RM");
- this.shouldUnregisterFlag.set(true);
- for (int i = 0 ; i < taskSchedulers.length ; i++) {
- if (this.taskSchedulers[i] != null) {
- this.taskSchedulers[i].setShouldUnregister();
- }
- }
- }
-
- public ContainerSignatureMatcher getContainerSignatureMatcher() {
- return containerSignatureMatcher;
- }
-
- public boolean hasUnregistered() {
- boolean result = true;
- for (int i = 0 ; i < taskSchedulers.length ; i++) {
- result = result & this.taskSchedulers[i].hasUnregistered();
- if (result == false) {
- return result;
- }
- }
- return result;
- }
-
- @VisibleForTesting
- public String getHistoryUrl() {
- Configuration config = this.appContext.getAMConf();
- String historyUrl = "";
-
- String loggingClass = config.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "");
- String historyUrlTemplate = config.get(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
- TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE_DEFAULT);
- String historyUrlBase = config.get(TezConfiguration.TEZ_HISTORY_URL_BASE, "");
-
-
- if (loggingClass.equals("org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService") &&
- !historyUrlTemplate.isEmpty() &&
- !historyUrlBase.isEmpty()) {
- // replace the placeholders, while tolerating extra or missing "/" in input. replace all
- // instances of consecutive "/" with single (except for the http(s):// case
- historyUrl = historyUrlTemplate
- .replaceAll(APPLICATION_ID_PLACEHOLDER, appContext.getApplicationID().toString())
- .replaceAll(HISTORY_URL_BASE, historyUrlBase)
- .replaceAll("([^:])/{2,}", "$1/");
-
- // make sure we have a valid scheme
- if (!historyUrl.startsWith("http")) {
- historyUrl = "http://" + historyUrl;
- }
- }
-
- return historyUrl;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
new file mode 100644
index 0000000..29143a2
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
@@ -0,0 +1,786 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.app.ServicePluginLifecycleAbstractService;
+import org.apache.tez.serviceplugins.api.TaskScheduler;
+import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
+import org.apache.tez.serviceplugins.api.TaskSchedulerContext.AppFinalStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.TaskLocationHint;
+import org.apache.tez.dag.api.TaskLocationHint.TaskBasedLocationAffinity;
+import org.apache.tez.dag.api.client.DAGClientServer;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.DAGAppMaster;
+import org.apache.tez.dag.app.DAGAppMasterState;
+import org.apache.tez.dag.app.dag.TaskAttempt;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.event.DAGAppMasterEvent;
+import org.apache.tez.dag.app.dag.event.DAGAppMasterEventSchedulingServiceError;
+import org.apache.tez.dag.app.dag.event.DAGAppMasterEventType;
+import org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdateTAAssigned;
+import org.apache.tez.dag.app.rm.container.AMContainer;
+import org.apache.tez.dag.app.rm.container.AMContainerEventAssignTA;
+import org.apache.tez.dag.app.rm.container.AMContainerEventCompleted;
+import org.apache.tez.dag.app.rm.container.AMContainerEventLaunchRequest;
+import org.apache.tez.dag.app.rm.container.AMContainerEventStopRequest;
+import org.apache.tez.dag.app.rm.container.AMContainerEventTASucceeded;
+import org.apache.tez.dag.app.rm.container.AMContainerState;
+import org.apache.tez.common.ContainerSignatureMatcher;
+import org.apache.tez.dag.app.rm.node.AMNodeEventContainerAllocated;
+import org.apache.tez.dag.app.rm.node.AMNodeEventNodeCountUpdated;
+import org.apache.tez.dag.app.rm.node.AMNodeEventStateChanged;
+import org.apache.tez.dag.app.rm.node.AMNodeEventTaskAttemptEnded;
+import org.apache.tez.dag.app.rm.node.AMNodeEventTaskAttemptSucceeded;
+import org.apache.tez.dag.app.web.WebUIService;
+import org.apache.tez.dag.records.TaskAttemptTerminationCause;
+
+import com.google.common.base.Preconditions;
+
+
+public class TaskSchedulerManager extends AbstractService implements
+ EventHandler<AMSchedulerEvent> {
+ static final Logger LOG = LoggerFactory.getLogger(TaskSchedulerManager.class);
+
+ static final String APPLICATION_ID_PLACEHOLDER = "__APPLICATION_ID__";
+ static final String HISTORY_URL_BASE = "__HISTORY_URL_BASE__";
+
+ protected final AppContext appContext;
+ @SuppressWarnings("rawtypes")
+ private final EventHandler eventHandler;
+ private final String historyUrl;
+ private DAGAppMaster dagAppMaster;
+ private Map<ApplicationAccessType, String> appAcls = null;
+ private Thread eventHandlingThread;
+ private volatile boolean stopEventHandling;
+ // Has a signal (SIGTERM etc) been issued?
+ protected volatile boolean isSignalled = false;
+ final DAGClientServer clientService;
+ private final ContainerSignatureMatcher containerSignatureMatcher;
+ private int cachedNodeCount = -1;
+ private AtomicBoolean shouldUnregisterFlag =
+ new AtomicBoolean(false);
+ private final WebUIService webUI;
+ private final NamedEntityDescriptor[] taskSchedulerDescriptors;
+ protected final TaskScheduler[]taskSchedulers;
+ protected final ServicePluginLifecycleAbstractService []taskSchedulerServiceWrappers;
+
+ // Single executor service shared by all Schedulers for context callbacks
+ @VisibleForTesting
+ final ExecutorService appCallbackExecutor;
+
+ private final boolean isPureLocalMode;
+ // If running in non local-only mode, the YARN task scheduler will always run to take care of
+ // registration with YARN and heartbeats to YARN.
+ // Splitting registration and heartbeats is not straight-forward due to the taskScheduler being
+ // tied to a ContainerRequestType.
+ // Custom AppIds to avoid container conflicts if there's multiple sources
+ private final long SCHEDULER_APP_ID_BASE = 111101111;
+ private final long SCHEDULER_APP_ID_INCREMENT = 111111111;
+
+ BlockingQueue<AMSchedulerEvent> eventQueue
+ = new LinkedBlockingQueue<AMSchedulerEvent>();
+
+ // Not tracking container / task to schedulerId. Instead relying on everything flowing through
+ // the system and being propagated back via events.
+
+ /**
+ *
+ * @param appContext
+ * @param clientService
+ * @param eventHandler
+ * @param containerSignatureMatcher
+ * @param webUI
+ * @param schedulerDescriptors the list of scheduler descriptors. Tez internal classes will not have the class names populated.
+ * An empty list defaults to using the YarnTaskScheduler as the only source.
+ * @param isPureLocalMode whether the AM is running in local mode
+ */
+ @SuppressWarnings("rawtypes")
+ public TaskSchedulerManager(AppContext appContext,
+ DAGClientServer clientService, EventHandler eventHandler,
+ ContainerSignatureMatcher containerSignatureMatcher,
+ WebUIService webUI,
+ List<NamedEntityDescriptor> schedulerDescriptors,
+ boolean isPureLocalMode) {
+ super(TaskSchedulerManager.class.getName());
+ Preconditions.checkArgument(schedulerDescriptors != null && !schedulerDescriptors.isEmpty(),
+ "TaskSchedulerDescriptors must be specified");
+ this.appContext = appContext;
+ this.eventHandler = eventHandler;
+ this.clientService = clientService;
+ this.containerSignatureMatcher = containerSignatureMatcher;
+ this.webUI = webUI;
+ this.historyUrl = getHistoryUrl();
+ this.isPureLocalMode = isPureLocalMode;
+ this.appCallbackExecutor = createAppCallbackExecutorService();
+ if (this.webUI != null) {
+ this.webUI.setHistoryUrl(this.historyUrl);
+ }
+
+ this.taskSchedulerDescriptors = schedulerDescriptors.toArray(new NamedEntityDescriptor[schedulerDescriptors.size()]);
+
+ taskSchedulers = new TaskScheduler[this.taskSchedulerDescriptors.length];
+ taskSchedulerServiceWrappers = new ServicePluginLifecycleAbstractService[this.taskSchedulerDescriptors.length];
+ }
+
+ public Map<ApplicationAccessType, String> getApplicationAcls() {
+ return appAcls;
+ }
+
+ public void setSignalled(boolean isSignalled) {
+ this.isSignalled = isSignalled;
+ LOG.info("TaskScheduler notified that iSignalled was : " + isSignalled);
+ }
+
+ public int getNumClusterNodes() {
+ return cachedNodeCount;
+ }
+
+ public Resource getAvailableResources(int schedulerId) {
+ return taskSchedulers[schedulerId].getAvailableResources();
+ }
+
+ public Resource getTotalResources(int schedulerId) {
+ return taskSchedulers[schedulerId].getTotalResources();
+ }
+
+ private ExecutorService createAppCallbackExecutorService() {
+ return Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("TaskSchedulerAppCallbackExecutor #%d")
+ .setDaemon(true)
+ .build());
+ }
+
+ public synchronized void handleEvent(AMSchedulerEvent sEvent) {
+ LOG.info("Processing the event " + sEvent.toString());
+ switch (sEvent.getType()) {
+ case S_TA_LAUNCH_REQUEST:
+ handleTaLaunchRequest((AMSchedulerEventTALaunchRequest) sEvent);
+ break;
+ case S_TA_ENDED: // TaskAttempt considered complete.
+ AMSchedulerEventTAEnded event = (AMSchedulerEventTAEnded)sEvent;
+ switch(event.getState()) {
+ case FAILED:
+ case KILLED:
+ handleTAUnsuccessfulEnd(event);
+ break;
+ case SUCCEEDED:
+ handleTASucceeded(event);
+ break;
+ default:
+ throw new TezUncheckedException("Unexecpted TA_ENDED state: " + event.getState());
+ }
+ break;
+ case S_CONTAINER_DEALLOCATE:
+ handleContainerDeallocate((AMSchedulerEventDeallocateContainer)sEvent);
+ break;
+ case S_NODE_UNBLACKLISTED:
+ // fall through
+ case S_NODE_BLACKLISTED:
+ handleNodeBlacklistUpdate((AMSchedulerEventNodeBlacklistUpdate)sEvent);
+ break;
+ case S_NODE_UNHEALTHY:
+ break;
+ case S_NODE_HEALTHY:
+ // Consider changing this to work like BLACKLISTING.
+ break;
+ default:
+ break;
+ }
+ }
+
+ @Override
+ public void handle(AMSchedulerEvent event) {
+ int qSize = eventQueue.size();
+ if (qSize != 0 && qSize % 1000 == 0) {
+ LOG.info("Size of event-queue in RMContainerAllocator is " + qSize);
+ }
+ int remCapacity = eventQueue.remainingCapacity();
+ if (remCapacity < 1000) {
+ LOG.warn("Very low remaining capacity in the event-queue "
+ + "of RMContainerAllocator: " + remCapacity);
+ }
+ try {
+ eventQueue.put(event);
+ } catch (InterruptedException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void sendEvent(Event<?> event) {
+ eventHandler.handle(event);
+ }
+
+ private void handleNodeBlacklistUpdate(AMSchedulerEventNodeBlacklistUpdate event) {
+ if (event.getType() == AMSchedulerEventType.S_NODE_BLACKLISTED) {
+ taskSchedulers[event.getSchedulerId()].blacklistNode(event.getNodeId());
+ } else if (event.getType() == AMSchedulerEventType.S_NODE_UNBLACKLISTED) {
+ taskSchedulers[event.getSchedulerId()].unblacklistNode(event.getNodeId());
+ } else {
+ throw new TezUncheckedException("Invalid event type: " + event.getType());
+ }
+ }
+
+ private void handleContainerDeallocate(
+ AMSchedulerEventDeallocateContainer event) {
+ ContainerId containerId = event.getContainerId();
+ // TODO what happens to the task that was connected to this container?
+ // current assumption is that it will eventually call handleTaStopRequest
+ //TaskAttempt taskAttempt = (TaskAttempt)
+ taskSchedulers[event.getSchedulerId()].deallocateContainer(containerId);
+ // TODO does this container need to be stopped via C_STOP_REQUEST
+ sendEvent(new AMContainerEventStopRequest(containerId));
+ }
+
+ private void handleTAUnsuccessfulEnd(AMSchedulerEventTAEnded event) {
+ TaskAttempt attempt = event.getAttempt();
+ // Propagate state and failure cause (if any) when informing the scheduler about the de-allocation.
+ boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()]
+ .deallocateTask(attempt, false, event.getTaskAttemptEndReason(), event.getDiagnostics());
+ // use stored value of container id in case the scheduler has removed this
+ // assignment because the task has been deallocated earlier.
+ // retroactive case
+ ContainerId attemptContainerId = attempt.getAssignedContainerID();
+
+ if(!wasContainerAllocated) {
+ LOG.info("Task: " + attempt.getID() +
+ " has no container assignment in the scheduler");
+ if (attemptContainerId != null) {
+ LOG.error("No container allocated to task: " + attempt.getID()
+ + " according to scheduler. Task reported container id: "
+ + attemptContainerId);
+ }
+ }
+
+ if (attemptContainerId != null) {
+ // TODO either ways send the necessary events
+ // Ask the container to stop.
+ sendEvent(new AMContainerEventStopRequest(attemptContainerId));
+ // Inform the Node - the task has asked to be STOPPED / has already
+ // stopped.
+ // AMNodeImpl blacklisting logic does not account for KILLED attempts.
+ sendEvent(new AMNodeEventTaskAttemptEnded(appContext.getAllContainers().
+ get(attemptContainerId).getContainer().getNodeId(), event.getSchedulerId(),
+ attemptContainerId,
+ attempt.getID(), event.getState() == TaskAttemptState.FAILED));
+ }
+ }
+
+ private void handleTASucceeded(AMSchedulerEventTAEnded event) {
+ TaskAttempt attempt = event.getAttempt();
+ ContainerId usedContainerId = event.getUsedContainerId();
+
+ // This could be null if a task fails / is killed before a container is
+ // assigned to it.
+ if (event.getUsedContainerId() != null) {
+ sendEvent(new AMContainerEventTASucceeded(usedContainerId,
+ event.getAttemptID()));
+ sendEvent(new AMNodeEventTaskAttemptSucceeded(appContext.getAllContainers().
+ get(usedContainerId).getContainer().getNodeId(), event.getSchedulerId(), usedContainerId,
+ event.getAttemptID()));
+ }
+
+ boolean wasContainerAllocated = taskSchedulers[event.getSchedulerId()].deallocateTask(attempt,
+ true, null, event.getDiagnostics());
+ if (!wasContainerAllocated) {
+ LOG.error("De-allocated successful task: " + attempt.getID()
+ + ", but TaskScheduler reported no container assigned to task");
+ }
+ }
+
+ private void handleTaLaunchRequest(AMSchedulerEventTALaunchRequest event) {
+ TaskAttempt taskAttempt = event.getTaskAttempt();
+ TaskLocationHint locationHint = event.getLocationHint();
+ String hosts[] = null;
+ String racks[] = null;
+ if (locationHint != null) {
+ TaskBasedLocationAffinity taskAffinity = locationHint.getAffinitizedTask();
+ if (taskAffinity != null) {
+ Vertex vertex = appContext.getCurrentDAG().getVertex(taskAffinity.getVertexName());
+ Preconditions.checkNotNull(vertex, "Invalid vertex in task based affinity " + taskAffinity
+ + " for attempt: " + taskAttempt.getID());
+ int taskIndex = taskAffinity.getTaskIndex();
+ Preconditions.checkState(taskIndex >=0 && taskIndex < vertex.getTotalTasks(),
+ "Invalid taskIndex in task based affinity " + taskAffinity
+ + " for attempt: " + taskAttempt.getID());
+ TaskAttempt affinityAttempt = vertex.getTask(taskIndex).getSuccessfulAttempt();
+ if (affinityAttempt != null) {
+ Preconditions.checkNotNull(affinityAttempt.getAssignedContainerID(), affinityAttempt.getID());
+ taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt,
+ event.getCapability(),
+ affinityAttempt.getAssignedContainerID(),
+ Priority.newInstance(event.getPriority()),
+ event.getContainerContext(),
+ event);
+ return;
+ }
+ LOG.info("Attempt: " + taskAttempt.getID() + " has task based affinity to " + taskAffinity
+ + " but no locality information exists for it. Ignoring hint.");
+ // fall through with null hosts/racks
+ } else {
+ hosts = (locationHint.getHosts() != null) ? locationHint
+ .getHosts().toArray(
+ new String[locationHint.getHosts().size()]) : null;
+ racks = (locationHint.getRacks() != null) ? locationHint.getRacks()
+ .toArray(new String[locationHint.getRacks().size()]) : null;
+ }
+ }
+
+ taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt,
+ event.getCapability(),
+ hosts,
+ racks,
+ Priority.newInstance(event.getPriority()),
+ event.getContainerContext(),
+ event);
+ }
+
+ @VisibleForTesting
+ TaskScheduler createTaskScheduler(String host, int port, String trackingUrl,
+ AppContext appContext,
+ NamedEntityDescriptor taskSchedulerDescriptor,
+ long customAppIdIdentifier,
+ int schedulerId) {
+ TaskSchedulerContext rawContext =
+ new TaskSchedulerContextImpl(this, appContext, schedulerId, trackingUrl,
+ customAppIdIdentifier, host, port, taskSchedulerDescriptor.getUserPayload());
+ TaskSchedulerContext wrappedContext = wrapTaskSchedulerContext(rawContext);
+ String schedulerName = taskSchedulerDescriptor.getEntityName();
+ if (schedulerName.equals(TezConstants.getTezYarnServicePluginName())) {
+ return createYarnTaskScheduler(wrappedContext, schedulerId);
+ } else if (schedulerName.equals(TezConstants.getTezUberServicePluginName())) {
+ return createUberTaskScheduler(wrappedContext, schedulerId);
+ } else {
+ return createCustomTaskScheduler(wrappedContext, taskSchedulerDescriptor, schedulerId);
+ }
+ }
+
+ @VisibleForTesting
+ TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
+ return new TaskSchedulerContextImplWrapper(rawContext, appCallbackExecutor);
+ }
+
+ @VisibleForTesting
+ TaskScheduler createYarnTaskScheduler(TaskSchedulerContext taskSchedulerContext,
+ int schedulerId) {
+ LOG.info("Creating TaskScheduler: YarnTaskSchedulerService");
+ return new YarnTaskSchedulerService(taskSchedulerContext);
+ }
+
+ @VisibleForTesting
+ TaskScheduler createUberTaskScheduler(TaskSchedulerContext taskSchedulerContext,
+ int schedulerId) {
+ LOG.info("Creating TaskScheduler: Local TaskScheduler");
+ return new LocalTaskSchedulerService(taskSchedulerContext);
+ }
+
+ @SuppressWarnings("unchecked")
+ TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext,
+ NamedEntityDescriptor taskSchedulerDescriptor,
+ int schedulerId) {
+ LOG.info("Creating custom TaskScheduler {}:{}", taskSchedulerDescriptor.getEntityName(),
+ taskSchedulerDescriptor.getClassName());
+ Class<? extends TaskScheduler> taskSchedulerClazz =
+ (Class<? extends TaskScheduler>) ReflectionUtils
+ .getClazz(taskSchedulerDescriptor.getClassName());
+ try {
+ Constructor<? extends TaskScheduler> ctor = taskSchedulerClazz
+ .getConstructor(TaskSchedulerContext.class);
+ return ctor.newInstance(taskSchedulerContext);
+ } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+
+ @VisibleForTesting
+ protected void instantiateSchedulers(String host, int port, String trackingUrl,
+ AppContext appContext) {
+ // Iterate over the list and create all the taskSchedulers
+ int j = 0;
+ for (int i = 0; i < taskSchedulerDescriptors.length; i++) {
+ long customAppIdIdentifier;
+ if (isPureLocalMode || taskSchedulerDescriptors[i].getEntityName().equals(
+ TezConstants.getTezYarnServicePluginName())) { // Use the app identifier from the appId.
+ customAppIdIdentifier = appContext.getApplicationID().getClusterTimestamp();
+ } else {
+ customAppIdIdentifier = SCHEDULER_APP_ID_BASE + (j++ * SCHEDULER_APP_ID_INCREMENT);
+ }
+ LOG.info("ClusterIdentifier for TaskScheduler [" + i + ":" + taskSchedulerDescriptors[i].getEntityName() + "]=" +
+ customAppIdIdentifier);
+ taskSchedulers[i] = createTaskScheduler(host, port,
+ trackingUrl, appContext, taskSchedulerDescriptors[i], customAppIdIdentifier, i);
+ taskSchedulerServiceWrappers[i] = new ServicePluginLifecycleAbstractService<>(taskSchedulers[i]);
+ }
+ }
+
+
+ @Override
+ public synchronized void serviceStart() {
+ InetSocketAddress serviceAddr = clientService.getBindAddress();
+ dagAppMaster = appContext.getAppMaster();
+ // if web service is enabled then set tracking url. else disable it (value = "").
+ // the actual url set on the rm web ui will be the proxy url set by WebAppProxyServlet, which
+ // always try to connect to AM and proxy the response. hence it wont work if the webUIService
+ // is not enabled.
+ String trackingUrl = (webUI != null) ? webUI.getTrackingURL() : "";
+ instantiateSchedulers(serviceAddr.getHostName(), serviceAddr.getPort(), trackingUrl, appContext);
+
+ for (int i = 0 ; i < taskSchedulers.length ; i++) {
+ taskSchedulerServiceWrappers[i].init(getConfig());
+ taskSchedulerServiceWrappers[i].start();
+ if (shouldUnregisterFlag.get()) {
+ // Flag may have been set earlier when task scheduler was not initialized
+ // External services could need to talk to some other entity.
+ taskSchedulers[i].setShouldUnregister();
+ }
+ }
+
+ this.eventHandlingThread = new Thread("TaskSchedulerEventHandlerThread") {
+ @Override
+ public void run() {
+
+ AMSchedulerEvent event;
+
+ while (!stopEventHandling && !Thread.currentThread().isInterrupted()) {
+ try {
+ if (TaskSchedulerManager.this.eventQueue.peek() == null) {
+ notifyForTest();
+ }
+ event = TaskSchedulerManager.this.eventQueue.take();
+ } catch (InterruptedException e) {
+ if(!stopEventHandling) {
+ LOG.warn("Continuing after interrupt : ", e);
+ }
+ continue;
+ }
+
+ try {
+ handleEvent(event);
+ } catch (Throwable t) {
+ LOG.error("Error in handling event type " + event.getType()
+ + " to the TaskScheduler", t);
+ // Kill the AM.
+ sendEvent(new DAGAppMasterEvent(DAGAppMasterEventType.INTERNAL_ERROR));
+ return;
+ } finally {
+ notifyForTest();
+ }
+ }
+ }
+ };
+ this.eventHandlingThread.start();
+ }
+
+ protected void notifyForTest() {
+ }
+
+ public void initiateStop() {
+ for (int i = 0 ; i < taskSchedulers.length ; i++) {
+ taskSchedulers[i].initiateStop();
+ }
+ }
+
+ @Override
+ public void serviceStop() throws InterruptedException {
+ synchronized(this) {
+ this.stopEventHandling = true;
+ if (eventHandlingThread != null)
+ eventHandlingThread.interrupt();
+ }
+ for (int i = 0 ; i < taskSchedulers.length ; i++) {
+ if (taskSchedulers[i] != null) {
+ taskSchedulerServiceWrappers[i].stop();
+ }
+ }
+ LOG.info("Shutting down AppCallbackExecutor");
+ appCallbackExecutor.shutdownNow();
+ appCallbackExecutor.awaitTermination(1000l, TimeUnit.MILLISECONDS);
+ }
+
+ // TaskSchedulerAppCallback methods with schedulerId, where relevant
+ public synchronized void taskAllocated(int schedulerId, Object task,
+ Object appCookie,
+ Container container) {
+ AMSchedulerEventTALaunchRequest event =
+ (AMSchedulerEventTALaunchRequest) appCookie;
+ ContainerId containerId = container.getId();
+ if (appContext.getAllContainers()
+ .addContainerIfNew(container, schedulerId, event.getLauncherId(),
+ event.getTaskCommId())) {
+ appContext.getNodeTracker().nodeSeen(container.getNodeId(), schedulerId);
+ sendEvent(new AMNodeEventContainerAllocated(container
+ .getNodeId(), schedulerId, container.getId()));
+ }
+
+
+ TaskAttempt taskAttempt = event.getTaskAttempt();
+ // TODO - perhaps check if the task still needs this container
+ // because the deallocateTask downcall may have raced with the
+ // taskAllocated() upcall
+ assert task.equals(taskAttempt);
+
+ if (appContext.getAllContainers().get(containerId).getState() == AMContainerState.ALLOCATED) {
+ sendEvent(new AMContainerEventLaunchRequest(containerId, taskAttempt.getVertexID(),
+ event.getContainerContext(), event.getLauncherId(), event.getTaskCommId()));
+ }
+ sendEvent(new DAGEventSchedulerUpdateTAAssigned(taskAttempt, container));
+ sendEvent(new AMContainerEventAssignTA(containerId, taskAttempt.getID(),
+ event.getRemoteTaskSpec(), event.getContainerContext().getLocalResources(), event
+ .getContainerContext().getCredentials(), event.getPriority()));
+ }
+
+ public synchronized void containerCompleted(int schedulerId, Object task, ContainerStatus containerStatus) {
+ // SchedulerId isn't used here since no node updates are sent out
+ // Inform the Containers about completion.
+ AMContainer amContainer = appContext.getAllContainers().get(containerStatus.getContainerId());
+ if (amContainer != null) {
+ String message = "Container completed. ";
+ TaskAttemptTerminationCause errCause = TaskAttemptTerminationCause.CONTAINER_EXITED;
+ int exitStatus = containerStatus.getExitStatus();
+ if (exitStatus == ContainerExitStatus.PREEMPTED) {
+ message = "Container preempted externally. ";
+ errCause = TaskAttemptTerminationCause.EXTERNAL_PREEMPTION;
+ } else if (exitStatus == ContainerExitStatus.DISKS_FAILED) {
+ message = "Container disk failed. ";
+ errCause = TaskAttemptTerminationCause.NODE_DISK_ERROR;
+ } else if (exitStatus != ContainerExitStatus.SUCCESS){
+ message = "Container failed, exitCode=" + exitStatus + ". ";
+ }
+ if (containerStatus.getDiagnostics() != null) {
+ message += containerStatus.getDiagnostics();
+ }
+ sendEvent(new AMContainerEventCompleted(amContainer.getContainerId(), exitStatus, message, errCause));
+ }
+ }
+
+ public synchronized void containerBeingReleased(int schedulerId, ContainerId containerId) {
+ // SchedulerId isn't used here since no node updates are sent out
+ AMContainer amContainer = appContext.getAllContainers().get(containerId);
+ if (amContainer != null) {
+ sendEvent(new AMContainerEventStopRequest(containerId));
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public synchronized void nodesUpdated(int schedulerId, List<NodeReport> updatedNodes) {
+ for (NodeReport nr : updatedNodes) {
+ // Scheduler will find out from the node, if at all.
+ // Relying on the RM to not allocate containers on an unhealthy node.
+ eventHandler.handle(new AMNodeEventStateChanged(nr, schedulerId));
+ }
+ }
+
+ public synchronized void appShutdownRequested(int schedulerId) {
+ // This can happen if the RM has been restarted. If it is in that state,
+ // this application must clean itself up.
+ LOG.info("App shutdown requested by scheduler {}", schedulerId);
+ sendEvent(new DAGAppMasterEvent(DAGAppMasterEventType.AM_REBOOT));
+ }
+
+ public synchronized void setApplicationRegistrationData(
+ int schedulerId,
+ Resource maxContainerCapability,
+ Map<ApplicationAccessType, String> appAcls,
+ ByteBuffer clientAMSecretKey) {
+ this.appContext.getClusterInfo().setMaxContainerCapability(
+ maxContainerCapability);
+ this.appAcls = appAcls;
+ this.clientService.setClientAMSecretKey(clientAMSecretKey);
+ }
+
+ // Not synchronized to avoid deadlocks from TaskScheduler callbacks.
+ // TaskScheduler uses a separate thread for it's callbacks. Since this method
+ // returns a value which is required, the TaskScheduler wait for the call to
+ // complete and can hence lead to a deadlock if called from within a TSEH lock.
+ public AppFinalStatus getFinalAppStatus() {
+ FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
+ StringBuffer sb = new StringBuffer();
+ if (dagAppMaster == null) {
+ finishState = FinalApplicationStatus.UNDEFINED;
+ sb.append("App not yet initialized");
+ } else {
+ DAGAppMasterState appMasterState = dagAppMaster.getState();
+ if (appMasterState == DAGAppMasterState.SUCCEEDED) {
+ finishState = FinalApplicationStatus.SUCCEEDED;
+ } else if (appMasterState == DAGAppMasterState.KILLED
+ || (appMasterState == DAGAppMasterState.RUNNING && isSignalled)) {
+ finishState = FinalApplicationStatus.KILLED;
+ } else if (appMasterState == DAGAppMasterState.FAILED
+ || appMasterState == DAGAppMasterState.ERROR) {
+ finishState = FinalApplicationStatus.FAILED;
+ } else {
+ finishState = FinalApplicationStatus.UNDEFINED;
+ }
+ List<String> diagnostics = dagAppMaster.getDiagnostics();
+ if(diagnostics != null) {
+ for (String s : diagnostics) {
+ sb.append(s).append("\n");
+ }
+ }
+ }
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Setting job diagnostics to " + sb.toString());
+ }
+
+ // if history url is set use the same, if historyUrl is set to "" then rm ui disables the
+ // history url
+ return new AppFinalStatus(finishState, sb.toString(), historyUrl);
+ }
+
+
+
+ // Not synchronized to avoid deadlocks from TaskScheduler callbacks.
+ // TaskScheduler uses a separate thread for it's callbacks. Since this method
+ // returns a value which is required, the TaskScheduler wait for the call to
+ // complete and can hence lead to a deadlock if called from within a TSEH lock.
+ public float getProgress(int schedulerId) {
+ // at this point allocate has been called and so node count must be available
+ // may change after YARN-1722
+ // This is a heartbeat in from the scheduler into the APP, and is being used to piggy-back and
+ // node updates from the cluster.
+
+ // Doubles as a mechanism to update node counts periodically. Hence schedulerId required.
+
+ // TODO Handle this in TEZ-2124. Need a way to know which scheduler is calling in.
+ int nodeCount = taskSchedulers[0].getClusterNodeCount();
+ if (nodeCount != cachedNodeCount) {
+ cachedNodeCount = nodeCount;
+ sendEvent(new AMNodeEventNodeCountUpdated(cachedNodeCount, schedulerId));
+ }
+ return dagAppMaster.getProgress();
+ }
+
+ public void onError(int schedulerId, Throwable t) {
+ LOG.info("Error reported by scheduler {} - {}", schedulerId, t);
+ sendEvent(new DAGAppMasterEventSchedulingServiceError(t));
+ }
+
+ public void dagCompleted() {
+ for (int i = 0 ; i < taskSchedulers.length ; i++) {
+ taskSchedulers[i].dagComplete();
+ }
+ }
+
+ public void dagSubmitted() {
+ // Nothing to do right now. Indicates that a new DAG has been submitted and
+ // the context has updated information.
+ }
+
+ public void preemptContainer(int schedulerId, ContainerId containerId) {
+ // TODO Why is this making a call back into the scheduler, when the call is originating from there.
+ // An AMContainer instance should already exist if an attempt is being made to preempt it
+ AMContainer amContainer = appContext.getAllContainers().get(containerId);
+ taskSchedulers[amContainer.getTaskSchedulerIdentifier()].deallocateContainer(containerId);
+ // Inform the Containers about completion.
+ sendEvent(new AMContainerEventCompleted(containerId, ContainerExitStatus.INVALID,
+ "Container preempted internally", TaskAttemptTerminationCause.INTERNAL_PREEMPTION));
+ }
+
+ public void setShouldUnregisterFlag() {
+ LOG.info("TaskScheduler notified that it should unregister from RM");
+ this.shouldUnregisterFlag.set(true);
+ for (int i = 0 ; i < taskSchedulers.length ; i++) {
+ if (this.taskSchedulers[i] != null) {
+ this.taskSchedulers[i].setShouldUnregister();
+ }
+ }
+ }
+
+ public ContainerSignatureMatcher getContainerSignatureMatcher() {
+ return containerSignatureMatcher;
+ }
+
+ public boolean hasUnregistered() {
+ boolean result = true;
+ for (int i = 0 ; i < taskSchedulers.length ; i++) {
+ result = result & this.taskSchedulers[i].hasUnregistered();
+ if (result == false) {
+ return result;
+ }
+ }
+ return result;
+ }
+
+ @VisibleForTesting
+ public String getHistoryUrl() {
+ Configuration config = this.appContext.getAMConf();
+ String historyUrl = "";
+
+ String loggingClass = config.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "");
+ String historyUrlTemplate = config.get(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
+ TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE_DEFAULT);
+ String historyUrlBase = config.get(TezConfiguration.TEZ_HISTORY_URL_BASE, "");
+
+
+ if (loggingClass.equals("org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService") &&
+ !historyUrlTemplate.isEmpty() &&
+ !historyUrlBase.isEmpty()) {
+ // replace the placeholders, while tolerating extra or missing "/" in input. replace all
+ // instances of consecutive "/" with single (except for the http(s):// case
+ historyUrl = historyUrlTemplate
+ .replaceAll(APPLICATION_ID_PLACEHOLDER, appContext.getApplicationID().toString())
+ .replaceAll(HISTORY_URL_BASE, historyUrlBase)
+ .replaceAll("([^:])/{2,}", "$1/");
+
+ // make sure we have a valid scheme
+ if (!historyUrl.startsWith("http")) {
+ historyUrl = "http://" + historyUrl;
+ }
+ }
+
+ return historyUrl;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 99cec2b..69c21d4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -50,15 +50,15 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerHeartbeatHandler;
import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.dag.event.DiagnosableEvent;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminatedBySystem;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminating;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventNodeFailed;
import org.apache.tez.dag.app.rm.AMSchedulerEventDeallocateContainer;
-import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
-import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
+import org.apache.tez.dag.app.rm.ContainerLauncherLaunchRequestEvent;
+import org.apache.tez.dag.app.rm.ContainerLauncherStopRequestEvent;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.HistoryEventHandler;
import org.apache.tez.dag.history.events.ContainerStoppedEvent;
@@ -81,7 +81,7 @@ public class AMContainerImpl implements AMContainer {
private final Container container;
private final AppContext appContext;
private final ContainerHeartbeatHandler containerHeartbeatHandler;
- private final TaskAttemptListener taskAttemptListener;
+ private final TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
protected final EventHandler eventHandler;
private final ContainerSignatureMatcher signatureMatcher;
private final int schedulerId;
@@ -308,7 +308,7 @@ public class AMContainerImpl implements AMContainer {
// Attempting to use a container based purely on reosurces required, etc needs
// additional change - JvmID, YarnChild, etc depend on TaskType.
public AMContainerImpl(Container container, ContainerHeartbeatHandler chh,
- TaskAttemptListener tal, ContainerSignatureMatcher signatureMatcher,
+ TaskCommunicatorManagerInterface tal, ContainerSignatureMatcher signatureMatcher,
AppContext appContext, int schedulerId, int launcherId, int taskCommId) {
ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
this.readLock = rwLock.readLock();
@@ -319,7 +319,7 @@ public class AMContainerImpl implements AMContainer {
this.signatureMatcher = signatureMatcher;
this.appContext = appContext;
this.containerHeartbeatHandler = chh;
- this.taskAttemptListener = tal;
+ this.taskCommunicatorManagerInterface = tal;
this.failedAssignments = new LinkedList<TezTaskAttemptID>();
this.schedulerId = schedulerId;
this.launcherId = launcherId;
@@ -466,7 +466,7 @@ public class AMContainerImpl implements AMContainer {
containerContext.getLocalResources(),
containerContext.getEnvironment(),
containerContext.getJavaOpts(),
- container.taskAttemptListener.getTaskCommunicator(container.taskCommId).getAddress(), containerContext.getCredentials(),
+ container.taskCommunicatorManagerInterface.getTaskCommunicator(container.taskCommId).getAddress(), containerContext.getCredentials(),
container.appContext, container.container.getResource(),
container.appContext.getAMConf());
@@ -1095,28 +1095,28 @@ public class AMContainerImpl implements AMContainer {
}
protected void sendStartRequestToNM(ContainerLaunchContext clc) {
- sendEvent(new NMCommunicatorLaunchRequestEvent(clc, container, launcherId, schedulerId, taskCommId));
+ sendEvent(new ContainerLauncherLaunchRequestEvent(clc, container, launcherId, schedulerId, taskCommId));
}
protected void sendStopRequestToNM() {
- sendEvent(new NMCommunicatorStopRequestEvent(containerId,
+ sendEvent(new ContainerLauncherStopRequestEvent(containerId,
container.getNodeId(), container.getContainerToken(), launcherId, schedulerId, taskCommId));
}
protected void unregisterAttemptFromListener(TezTaskAttemptID attemptId, TaskAttemptEndReason endReason, String diagnostics) {
- taskAttemptListener.unregisterTaskAttempt(attemptId, taskCommId, endReason, diagnostics);
+ taskCommunicatorManagerInterface.unregisterTaskAttempt(attemptId, taskCommId, endReason, diagnostics);
}
protected void registerAttemptWithListener(AMContainerTask amContainerTask) {
- taskAttemptListener.registerTaskAttempt(amContainerTask, this.containerId, taskCommId);
+ taskCommunicatorManagerInterface.registerTaskAttempt(amContainerTask, this.containerId, taskCommId);
}
protected void registerWithTAListener() {
- taskAttemptListener.registerRunningContainer(containerId, taskCommId);
+ taskCommunicatorManagerInterface.registerRunningContainer(containerId, taskCommId);
}
protected void unregisterFromTAListener(ContainerEndReason endReason, String diagnostics) {
- this.taskAttemptListener.unregisterRunningContainer(containerId, taskCommId, endReason, diagnostics);
+ this.taskCommunicatorManagerInterface.unregisterRunningContainer(containerId, taskCommId, endReason, diagnostics);
}
protected void registerWithContainerListener() {
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
index fcb9eaf..ab43db1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
@@ -31,19 +31,19 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerHeartbeatHandler;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
public class AMContainerMap extends AbstractService implements EventHandler<AMContainerEvent> {
private static final Logger LOG = LoggerFactory.getLogger(AMContainerMap.class);
private final ContainerHeartbeatHandler chh;
- private final TaskAttemptListener tal;
+ private final TaskCommunicatorManagerInterface tal;
private final AppContext context;
private final ContainerSignatureMatcher containerSignatureMatcher;
private final ConcurrentHashMap<ContainerId, AMContainer> containerMap;
- public AMContainerMap(ContainerHeartbeatHandler chh, TaskAttemptListener tal,
+ public AMContainerMap(ContainerHeartbeatHandler chh, TaskCommunicatorManagerInterface tal,
ContainerSignatureMatcher containerSignatureMatcher, AppContext context) {
super("AMContainerMaps");
this.chh = chh;
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
index b09eb86..3cab2da 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java
@@ -60,8 +60,8 @@ import org.apache.tez.common.ContainerTask;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.app.launcher.ContainerLauncherRouter;
-import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
+import org.apache.tez.dag.app.launcher.ContainerLauncherManager;
+import org.apache.tez.dag.app.rm.ContainerLauncherEvent;
import org.apache.tez.dag.history.HistoryEventHandler;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
@@ -122,13 +122,13 @@ public class MockDAGAppMaster extends DAGAppMaster {
// It can be used to preempt the container for a given task
public class MockContainerLauncher extends ContainerLauncher implements Runnable {
- BlockingQueue<NMCommunicatorEvent> eventQueue = new LinkedBlockingQueue<NMCommunicatorEvent>();
+ BlockingQueue<ContainerLauncherEvent> eventQueue = new LinkedBlockingQueue<ContainerLauncherEvent>();
Thread eventHandlingThread;
ListeningExecutorService executorService;
Map<ContainerId, ContainerData> containers = Maps.newConcurrentMap();
ArrayBlockingQueue<Worker> workers;
- TaskAttemptListenerImpTezDag taListener;
+ TaskCommunicatorManager taskCommunicatorManager;
TezTaskCommunicatorImpl taskCommunicator;
AtomicBoolean startScheduling = new AtomicBoolean(true);
@@ -187,8 +187,8 @@ public class MockDAGAppMaster extends DAGAppMaster {
@Override
public void start() throws Exception {
- taListener = (TaskAttemptListenerImpTezDag) getTaskAttemptListener();
- taskCommunicator = (TezTaskCommunicatorImpl) taListener.getTaskCommunicator(0);
+ taskCommunicatorManager = (TaskCommunicatorManager) getTaskCommunicatorManager();
+ taskCommunicator = (TezTaskCommunicatorImpl) taskCommunicatorManager.getTaskCommunicator(0);
eventHandlingThread = new Thread(this);
eventHandlingThread.start();
ExecutorService rawExecutor = Executors.newFixedThreadPool(handlerConcurrency,
@@ -256,7 +256,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
}
public void preemptContainer(ContainerData cData) {
- getTaskSchedulerEventHandler().containerCompleted(0, null,
+ getTaskSchedulerManager().containerCompleted(0, null,
ContainerStatus.newInstance(cData.cId, null, "Preempted", ContainerExitStatus.PREEMPTED));
cData.clear();
}
@@ -495,7 +495,7 @@ public class MockDAGAppMaster extends DAGAppMaster {
throw new TezUncheckedException(e);
}
containerLauncherContext =
- new ContainerLauncherContextImpl(getContext(), getTaskAttemptListener(), userPayload);
+ new ContainerLauncherContextImpl(getContext(), getTaskCommunicatorManager(), userPayload);
containerLauncher = new MockContainerLauncher(launcherGoFlag, containerLauncherContext);
shutdownHandler = new MockDAGAppMasterShutdownHandler();
this.initFailFlag = initFailFlag;
@@ -507,10 +507,11 @@ public class MockDAGAppMaster extends DAGAppMaster {
// use mock container launcher for tests
@Override
- protected ContainerLauncherRouter createContainerLauncherRouter(List<NamedEntityDescriptor> containerLauncherDescirptors,
- boolean isLocal)
+ protected ContainerLauncherManager createContainerLauncherManager(
+ List<NamedEntityDescriptor> containerLauncherDescirptors,
+ boolean isLocal)
throws UnknownHostException {
- return new ContainerLauncherRouter(containerLauncher, getContext());
+ return new ContainerLauncherManager(containerLauncher, getContext());
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
index 7584b4c..b0bc571 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
@@ -200,7 +200,7 @@ public class TestMockDAGAppMaster {
mockLauncher.waitTillContainersLaunched();
ContainerData cData = mockLauncher.getContainers().values().iterator().next();
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
- mockApp.getTaskSchedulerEventHandler().preemptContainer(0, cData.cId);
+ mockApp.getTaskSchedulerManager().preemptContainer(0, cData.cId);
mockLauncher.startScheduling(true);
dagClient.waitForCompletion();
[3/7] tez git commit: TEZ-2708. Rename classes and variables post
TEZ-2003 changes. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 6ee741a..a8ba445 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -117,7 +117,7 @@ import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerHeartbeatHandler;
import org.apache.tez.dag.app.MockClock;
import org.apache.tez.dag.app.TaskAttemptEventInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.RootInputInitializerManager;
@@ -153,7 +153,7 @@ import org.apache.tez.dag.app.dag.event.VertexEventType;
import org.apache.tez.dag.app.dag.impl.AMUserCodeException.Source;
import org.apache.tez.dag.app.dag.impl.DAGImpl.VertexGroupInfo;
import org.apache.tez.dag.app.dag.impl.TestVertexImpl.VertexManagerWithException.VMExceptionLocation;
-import org.apache.tez.dag.app.rm.TaskSchedulerEventHandler;
+import org.apache.tez.dag.app.rm.TaskSchedulerManager;
import org.apache.tez.dag.app.rm.container.AMContainerMap;
import org.apache.tez.dag.app.rm.container.ContainerContextMatcher;
import org.apache.tez.dag.history.DAGHistoryEvent;
@@ -161,7 +161,6 @@ import org.apache.tez.dag.history.HistoryEventHandler;
import org.apache.tez.dag.history.HistoryEventType;
import org.apache.tez.dag.library.vertexmanager.InputReadyVertexManager;
import org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager;
-import org.apache.tez.dag.records.TaskAttemptIdentifierImpl;
import org.apache.tez.dag.records.TaskAttemptTerminationCause;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -228,7 +227,7 @@ public class TestVertexImpl {
private Map<String, VertexImpl> vertices;
private Map<TezVertexID, VertexImpl> vertexIdMap;
private DrainDispatcher dispatcher;
- private TaskAttemptListener taskAttemptListener;
+ private TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
private Clock clock = new SystemClock();
private TaskHeartbeatHandler thh;
private AppContext appContext;
@@ -2077,16 +2076,16 @@ public class TestVertexImpl {
if (useCustomInitializer) {
if (customInitializer == null) {
v = new VertexImplWithControlledInitializerManager(vertexId, vPlan, vPlan.getName(), conf,
- dispatcher.getEventHandler(), taskAttemptListener,
+ dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
clock, thh, appContext, locationHint, dispatcher, updateTracker);
} else {
v = new VertexImplWithRunningInputInitializer(vertexId, vPlan, vPlan.getName(), conf,
- dispatcher.getEventHandler(), taskAttemptListener,
+ dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
clock, thh, appContext, locationHint, dispatcher, customInitializer, updateTracker);
}
} else {
v = new VertexImpl(vertexId, vPlan, vPlan.getName(), conf,
- dispatcher.getEventHandler(), taskAttemptListener,
+ dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
clock, thh, true, appContext, locationHint, vertexGroups, taskSpecificLaunchCmdOption,
updateTracker);
}
@@ -2162,7 +2161,7 @@ public class TestVertexImpl {
appContext = mock(AppContext.class);
thh = mock(TaskHeartbeatHandler.class);
historyEventHandler = mock(HistoryEventHandler.class);
- TaskSchedulerEventHandler taskScheduler = mock(TaskSchedulerEventHandler.class);
+ TaskSchedulerManager taskScheduler = mock(TaskSchedulerManager.class);
UserGroupInformation ugi;
try {
ugi = UserGroupInformation.getCurrentUser();
@@ -3227,7 +3226,7 @@ public class TestVertexImpl {
when(container.getNodeId()).thenReturn(nid);
when(container.getNodeHttpAddress()).thenReturn("localhost:0");
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appContext);
containers.addContainerIfNew(container, 0, 0, 0);
doReturn(containers).when(appContext).getAllContainers();
@@ -3262,7 +3261,7 @@ public class TestVertexImpl {
when(container.getNodeId()).thenReturn(nid);
when(container.getNodeHttpAddress()).thenReturn("localhost:0");
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appContext);
containers.addContainerIfNew(container, 0, 0, 0);
doReturn(containers).when(appContext).getAllContainers();
@@ -3298,7 +3297,7 @@ public class TestVertexImpl {
when(container.getNodeId()).thenReturn(nid);
when(container.getNodeHttpAddress()).thenReturn("localhost:0");
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appContext);
containers.addContainerIfNew(container, 0, 0, 0);
doReturn(containers).when(appContext).getAllContainers();
@@ -5179,7 +5178,7 @@ public class TestVertexImpl {
vId = TezVertexID.getInstance(invalidDagId, 1);
VertexPlan vPlan = invalidDagPlan.getVertex(0);
VertexImpl v = new VertexImpl(vId, vPlan, vPlan.getName(), conf,
- dispatcher.getEventHandler(), taskAttemptListener,
+ dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
clock, thh, true, appContext, vertexLocationHint, null, taskSpecificLaunchCmdOption,
updateTracker);
v.setInputVertices(new HashMap());
@@ -5208,7 +5207,7 @@ public class TestVertexImpl {
VertexPlan vertexPlan, String vertexName,
Configuration conf,
EventHandler eventHandler,
- TaskAttemptListener taskAttemptListener,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
Clock clock, TaskHeartbeatHandler thh,
AppContext appContext,
VertexLocationHint vertexLocationHint,
@@ -5216,7 +5215,7 @@ public class TestVertexImpl {
InputInitializer presetInitializer,
StateChangeNotifier updateTracker) {
super(vertexId, vertexPlan, vertexName, conf, eventHandler,
- taskAttemptListener, clock, thh, true,
+ taskCommunicatorManagerInterface, clock, thh, true,
appContext, vertexLocationHint, null, taskSpecificLaunchCmdOption,
updateTracker);
this.presetInitializer = presetInitializer;
@@ -5248,14 +5247,14 @@ public class TestVertexImpl {
VertexPlan vertexPlan, String vertexName,
Configuration conf,
EventHandler eventHandler,
- TaskAttemptListener taskAttemptListener,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
Clock clock, TaskHeartbeatHandler thh,
AppContext appContext,
VertexLocationHint vertexLocationHint,
DrainDispatcher dispatcher,
StateChangeNotifier updateTracker) {
super(vertexId, vertexPlan, vertexName, conf, eventHandler,
- taskAttemptListener, clock, thh, true,
+ taskCommunicatorManagerInterface, clock, thh, true,
appContext, vertexLocationHint, null, taskSpecificLaunchCmdOption,
updateTracker);
this.dispatcher = dispatcher;
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
index 0e34f68..8bd288a 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
@@ -54,7 +54,7 @@ import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerContext;
import org.apache.tez.dag.app.DAGAppMaster;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.StateChangeNotifier;
@@ -517,7 +517,7 @@ public class TestVertexImpl2 {
vertex =
new VertexImpl(TezVertexID.fromString("vertex_1418197758681_0001_1_00"), vertexPlan,
- "testvertex", conf, mock(EventHandler.class), mock(TaskAttemptListener.class),
+ "testvertex", conf, mock(EventHandler.class), mock(TaskCommunicatorManagerInterface.class),
mock(Clock.class), mock(TaskHeartbeatHandler.class), false, mockAppContext,
VertexLocationHint.create(new LinkedList<TaskLocationHint>()), null,
new TaskSpecificLaunchCmdOption(conf), mock(StateChangeNotifier.class));
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
index 0f532fb..e389d64 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
@@ -55,7 +55,7 @@ import org.apache.tez.dag.api.records.DAGProtos.TezEntityDescriptorProto;
import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ClusterInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.DAGState;
import org.apache.tez.dag.app.dag.Task;
@@ -434,7 +434,7 @@ public class TestVertexRecovery {
DAGPlan dagPlan = createDAGPlan();
dag =
new DAGImpl(dagId, new Configuration(), dagPlan,
- dispatcher.getEventHandler(), mock(TaskAttemptListener.class),
+ dispatcher.getEventHandler(), mock(TaskCommunicatorManagerInterface.class),
new Credentials(), new SystemClock(), user,
mock(TaskHeartbeatHandler.class), mockAppContext);
when(mockAppContext.getCurrentDAG()).thenReturn(dag);
@@ -544,7 +544,7 @@ public class TestVertexRecovery {
DAGPlan dagPlan = createDAGPlanSingleVertex();
dag =
new DAGImpl(dagId, new Configuration(), dagPlan,
- dispatcher.getEventHandler(), mock(TaskAttemptListener.class),
+ dispatcher.getEventHandler(), mock(TaskCommunicatorManagerInterface.class),
new Credentials(), new SystemClock(), user,
mock(TaskHeartbeatHandler.class), mockAppContext);
when(mockAppContext.getCurrentDAG()).thenReturn(dag);
@@ -924,7 +924,7 @@ public class TestVertexRecovery {
DAGPlan dagPlan = createDAGPlanMR();
dag =
new DAGImpl(dagId, new Configuration(), dagPlan,
- dispatcher.getEventHandler(), mock(TaskAttemptListener.class),
+ dispatcher.getEventHandler(), mock(TaskCommunicatorManagerInterface.class),
new Credentials(), new SystemClock(), user,
mock(TaskHeartbeatHandler.class), mockAppContext);
when(mockAppContext.getCurrentDAG()).thenReturn(dag);
@@ -965,7 +965,7 @@ public class TestVertexRecovery {
DAGPlan dagPlan = createDAGPlan();
dag =
new DAGImpl(dagId, new Configuration(), dagPlan,
- dispatcher.getEventHandler(), mock(TaskAttemptListener.class),
+ dispatcher.getEventHandler(), mock(TaskCommunicatorManagerInterface.class),
new Credentials(), new SystemClock(), user,
mock(TaskHeartbeatHandler.class), mockAppContext);
when(mockAppContext.getCurrentDAG()).thenReturn(dag);
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java
new file mode 100644
index 0000000..4b931d4
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.launcher;
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
+import org.apache.tez.dag.app.rm.ContainerLauncherLaunchRequestEvent;
+import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
+import org.apache.tez.serviceplugins.api.ContainerLauncher;
+import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
+import org.apache.tez.serviceplugins.api.ContainerStopRequest;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+public class TestContainerLauncherManager {
+
+ @Before
+ @After
+ public void reset() {
+ ContainerLaucherRouterForMultipleLauncherTest.reset();
+ }
+
+ @Test(timeout = 5000)
+ public void testNoLaunchersSpecified() throws IOException {
+
+ AppContext appContext = mock(AppContext.class);
+ TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
+
+ try {
+
+ new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, null,
+ false);
+ fail("Expecting a failure without any launchers being specified");
+ } catch (IllegalArgumentException e) {
+
+ }
+ }
+
+ @Test(timeout = 5000)
+ public void testCustomLauncherSpecified() throws IOException {
+ Configuration conf = new Configuration(false);
+
+ AppContext appContext = mock(AppContext.class);
+ TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
+
+ String customLauncherName = "customLauncher";
+ List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
+ ByteBuffer bb = ByteBuffer.allocate(4);
+ bb.putInt(0, 3);
+ UserPayload customPayload = UserPayload.create(bb);
+ launcherDescriptors.add(
+ new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
+ .setUserPayload(customPayload));
+
+ ContainerLaucherRouterForMultipleLauncherTest clr =
+ new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
+ launcherDescriptors,
+ true);
+ try {
+ clr.init(conf);
+ clr.start();
+
+ assertEquals(1, clr.getNumContainerLaunchers());
+ assertFalse(clr.getYarnContainerLauncherCreated());
+ assertFalse(clr.getUberContainerLauncherCreated());
+ assertEquals(customLauncherName, clr.getContainerLauncherName(0));
+ assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
+ } finally {
+ clr.stop();
+ }
+ }
+
+ @Test(timeout = 5000)
+ public void testMultipleContainerLaunchers() throws IOException {
+ Configuration conf = new Configuration(false);
+ conf.set("testkey", "testvalue");
+ UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+
+ AppContext appContext = mock(AppContext.class);
+ TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
+
+ String customLauncherName = "customLauncher";
+ List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
+ ByteBuffer bb = ByteBuffer.allocate(4);
+ bb.putInt(0, 3);
+ UserPayload customPayload = UserPayload.create(bb);
+ launcherDescriptors.add(
+ new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
+ .setUserPayload(customPayload));
+ launcherDescriptors
+ .add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+ .setUserPayload(userPayload));
+
+ ContainerLaucherRouterForMultipleLauncherTest clr =
+ new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
+ launcherDescriptors,
+ true);
+ try {
+ clr.init(conf);
+ clr.start();
+
+ assertEquals(2, clr.getNumContainerLaunchers());
+ assertTrue(clr.getYarnContainerLauncherCreated());
+ assertFalse(clr.getUberContainerLauncherCreated());
+ assertEquals(customLauncherName, clr.getContainerLauncherName(0));
+ assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
+
+ assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
+ Configuration confParsed = TezUtils
+ .createConfFromUserPayload(clr.getContainerLauncherContext(1).getInitialUserPayload());
+ assertEquals("testvalue", confParsed.get("testkey"));
+ } finally {
+ clr.stop();
+ }
+ }
+
+ @Test(timeout = 5000)
+ public void testEventRouting() throws Exception {
+ Configuration conf = new Configuration(false);
+ UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+
+ AppContext appContext = mock(AppContext.class);
+ TaskCommunicatorManagerInterface tal = mock(TaskCommunicatorManagerInterface.class);
+
+ String customLauncherName = "customLauncher";
+ List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
+ ByteBuffer bb = ByteBuffer.allocate(4);
+ bb.putInt(0, 3);
+ UserPayload customPayload = UserPayload.create(bb);
+ launcherDescriptors.add(
+ new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
+ .setUserPayload(customPayload));
+ launcherDescriptors
+ .add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+ .setUserPayload(userPayload));
+
+ ContainerLaucherRouterForMultipleLauncherTest clr =
+ new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
+ launcherDescriptors,
+ true);
+ try {
+ clr.init(conf);
+ clr.start();
+
+ assertEquals(2, clr.getNumContainerLaunchers());
+ assertTrue(clr.getYarnContainerLauncherCreated());
+ assertFalse(clr.getUberContainerLauncherCreated());
+ assertEquals(customLauncherName, clr.getContainerLauncherName(0));
+ assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
+
+ verify(clr.getTestContainerLauncher(0)).initialize();
+ verify(clr.getTestContainerLauncher(0)).start();
+ verify(clr.getTestContainerLauncher(1)).initialize();
+ verify(clr.getTestContainerLauncher(1)).start();
+
+ ContainerLaunchContext clc1 = mock(ContainerLaunchContext.class);
+ Container container1 = mock(Container.class);
+
+ ContainerLaunchContext clc2 = mock(ContainerLaunchContext.class);
+ Container container2 = mock(Container.class);
+
+ ContainerLauncherLaunchRequestEvent launchRequestEvent1 =
+ new ContainerLauncherLaunchRequestEvent(clc1, container1, 0, 0, 0);
+ ContainerLauncherLaunchRequestEvent launchRequestEvent2 =
+ new ContainerLauncherLaunchRequestEvent(clc2, container2, 1, 0, 0);
+
+ clr.handle(launchRequestEvent1);
+
+
+ ArgumentCaptor<ContainerLaunchRequest> captor =
+ ArgumentCaptor.forClass(ContainerLaunchRequest.class);
+ verify(clr.getTestContainerLauncher(0)).launchContainer(captor.capture());
+ assertEquals(1, captor.getAllValues().size());
+ ContainerLaunchRequest launchRequest1 = captor.getValue();
+ assertEquals(clc1, launchRequest1.getContainerLaunchContext());
+
+ clr.handle(launchRequestEvent2);
+ captor = ArgumentCaptor.forClass(ContainerLaunchRequest.class);
+ verify(clr.getTestContainerLauncher(1)).launchContainer(captor.capture());
+ assertEquals(1, captor.getAllValues().size());
+ ContainerLaunchRequest launchRequest2 = captor.getValue();
+ assertEquals(clc2, launchRequest2.getContainerLaunchContext());
+
+ } finally {
+ clr.stop();
+ verify(clr.getTestContainerLauncher(0)).shutdown();
+ verify(clr.getTestContainerLauncher(1)).shutdown();
+ }
+ }
+
+ private static class ContainerLaucherRouterForMultipleLauncherTest
+ extends ContainerLauncherManager {
+
+ // All variables setup as static since methods being overridden are invoked by the ContainerLauncherRouter ctor,
+ // and regular variables will not be initialized at this point.
+ private static final AtomicInteger numContainerLaunchers = new AtomicInteger(0);
+ private static final Set<Integer> containerLauncherIndices = new HashSet<>();
+ private static final ContainerLauncher yarnContainerLauncher = mock(ContainerLauncher.class);
+ private static final ContainerLauncher uberContainerlauncher = mock(ContainerLauncher.class);
+ private static final AtomicBoolean yarnContainerLauncherCreated = new AtomicBoolean(false);
+ private static final AtomicBoolean uberContainerLauncherCreated = new AtomicBoolean(false);
+
+ private static final List<ContainerLauncherContext> containerLauncherContexts =
+ new LinkedList<>();
+ private static final List<String> containerLauncherNames = new LinkedList<>();
+ private static final List<ContainerLauncher> testContainerLaunchers = new LinkedList<>();
+
+
+ public static void reset() {
+ numContainerLaunchers.set(0);
+ containerLauncherIndices.clear();
+ yarnContainerLauncherCreated.set(false);
+ uberContainerLauncherCreated.set(false);
+ containerLauncherContexts.clear();
+ containerLauncherNames.clear();
+ testContainerLaunchers.clear();
+ }
+
+ public ContainerLaucherRouterForMultipleLauncherTest(AppContext context,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+ String workingDirectory,
+ List<NamedEntityDescriptor> containerLauncherDescriptors,
+ boolean isPureLocalMode) throws
+ UnknownHostException {
+ super(context, taskCommunicatorManagerInterface, workingDirectory,
+ containerLauncherDescriptors, isPureLocalMode);
+ }
+
+ @Override
+ ContainerLauncher createContainerLauncher(NamedEntityDescriptor containerLauncherDescriptor,
+ AppContext context,
+ ContainerLauncherContext containerLauncherContext,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+ String workingDirectory,
+ int containerLauncherIndex,
+ boolean isPureLocalMode) {
+ numContainerLaunchers.incrementAndGet();
+ boolean added = containerLauncherIndices.add(containerLauncherIndex);
+ assertTrue("Cannot add multiple launchers with the same index", added);
+ containerLauncherNames.add(containerLauncherDescriptor.getEntityName());
+ containerLauncherContexts.add(containerLauncherContext);
+ return super
+ .createContainerLauncher(containerLauncherDescriptor, context, containerLauncherContext,
+ taskCommunicatorManagerInterface, workingDirectory, containerLauncherIndex, isPureLocalMode);
+ }
+
+ @Override
+ ContainerLauncher createYarnContainerLauncher(
+ ContainerLauncherContext containerLauncherContext) {
+ yarnContainerLauncherCreated.set(true);
+ testContainerLaunchers.add(yarnContainerLauncher);
+ return yarnContainerLauncher;
+ }
+
+ @Override
+ ContainerLauncher createUberContainerLauncher(ContainerLauncherContext containerLauncherContext,
+ AppContext context,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
+ String workingDirectory,
+ boolean isPureLocalMode) {
+ uberContainerLauncherCreated.set(true);
+ testContainerLaunchers.add(uberContainerlauncher);
+ return uberContainerlauncher;
+ }
+
+ @Override
+ ContainerLauncher createCustomContainerLauncher(
+ ContainerLauncherContext containerLauncherContext,
+ NamedEntityDescriptor containerLauncherDescriptor) {
+ ContainerLauncher spyLauncher = spy(super.createCustomContainerLauncher(
+ containerLauncherContext, containerLauncherDescriptor));
+ testContainerLaunchers.add(spyLauncher);
+ return spyLauncher;
+ }
+
+ public int getNumContainerLaunchers() {
+ return numContainerLaunchers.get();
+ }
+
+ public boolean getYarnContainerLauncherCreated() {
+ return yarnContainerLauncherCreated.get();
+ }
+
+ public boolean getUberContainerLauncherCreated() {
+ return uberContainerLauncherCreated.get();
+ }
+
+ public String getContainerLauncherName(int containerLauncherIndex) {
+ return containerLauncherNames.get(containerLauncherIndex);
+ }
+
+ public ContainerLauncher getTestContainerLauncher(int containerLauncherIndex) {
+ return testContainerLaunchers.get(containerLauncherIndex);
+ }
+
+ public ContainerLauncherContext getContainerLauncherContext(int containerLauncherIndex) {
+ return containerLauncherContexts.get(containerLauncherIndex);
+ }
+ }
+
+ private static class FakeContainerLauncher extends ContainerLauncher {
+
+ public FakeContainerLauncher(
+ ContainerLauncherContext containerLauncherContext) {
+ super(containerLauncherContext);
+ }
+
+ @Override
+ public void launchContainer(ContainerLaunchRequest launchRequest) {
+
+ }
+
+ @Override
+ public void stopContainer(ContainerStopRequest stopRequest) {
+
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherRouter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherRouter.java b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherRouter.java
deleted file mode 100644
index d0caf8c..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherRouter.java
+++ /dev/null
@@ -1,359 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.launcher;
-
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
-
-import java.io.IOException;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
-import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
-import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
-import org.apache.tez.serviceplugins.api.ContainerLauncher;
-import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
-import org.apache.tez.serviceplugins.api.ContainerStopRequest;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-
-public class TestContainerLauncherRouter {
-
- @Before
- @After
- public void reset() {
- ContainerLaucherRouterForMultipleLauncherTest.reset();
- }
-
- @Test(timeout = 5000)
- public void testNoLaunchersSpecified() throws IOException {
-
- AppContext appContext = mock(AppContext.class);
- TaskAttemptListener tal = mock(TaskAttemptListener.class);
-
- try {
-
- new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null, null,
- false);
- fail("Expecting a failure without any launchers being specified");
- } catch (IllegalArgumentException e) {
-
- }
- }
-
- @Test(timeout = 5000)
- public void testCustomLauncherSpecified() throws IOException {
- Configuration conf = new Configuration(false);
-
- AppContext appContext = mock(AppContext.class);
- TaskAttemptListener tal = mock(TaskAttemptListener.class);
-
- String customLauncherName = "customLauncher";
- List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
- ByteBuffer bb = ByteBuffer.allocate(4);
- bb.putInt(0, 3);
- UserPayload customPayload = UserPayload.create(bb);
- launcherDescriptors.add(
- new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
- .setUserPayload(customPayload));
-
- ContainerLaucherRouterForMultipleLauncherTest clr =
- new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
- launcherDescriptors,
- true);
- try {
- clr.init(conf);
- clr.start();
-
- assertEquals(1, clr.getNumContainerLaunchers());
- assertFalse(clr.getYarnContainerLauncherCreated());
- assertFalse(clr.getUberContainerLauncherCreated());
- assertEquals(customLauncherName, clr.getContainerLauncherName(0));
- assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
- } finally {
- clr.stop();
- }
- }
-
- @Test(timeout = 5000)
- public void testMultipleContainerLaunchers() throws IOException {
- Configuration conf = new Configuration(false);
- conf.set("testkey", "testvalue");
- UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
-
- AppContext appContext = mock(AppContext.class);
- TaskAttemptListener tal = mock(TaskAttemptListener.class);
-
- String customLauncherName = "customLauncher";
- List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
- ByteBuffer bb = ByteBuffer.allocate(4);
- bb.putInt(0, 3);
- UserPayload customPayload = UserPayload.create(bb);
- launcherDescriptors.add(
- new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
- .setUserPayload(customPayload));
- launcherDescriptors
- .add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
- .setUserPayload(userPayload));
-
- ContainerLaucherRouterForMultipleLauncherTest clr =
- new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
- launcherDescriptors,
- true);
- try {
- clr.init(conf);
- clr.start();
-
- assertEquals(2, clr.getNumContainerLaunchers());
- assertTrue(clr.getYarnContainerLauncherCreated());
- assertFalse(clr.getUberContainerLauncherCreated());
- assertEquals(customLauncherName, clr.getContainerLauncherName(0));
- assertEquals(bb, clr.getContainerLauncherContext(0).getInitialUserPayload().getPayload());
-
- assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
- Configuration confParsed = TezUtils
- .createConfFromUserPayload(clr.getContainerLauncherContext(1).getInitialUserPayload());
- assertEquals("testvalue", confParsed.get("testkey"));
- } finally {
- clr.stop();
- }
- }
-
- @Test(timeout = 5000)
- public void testEventRouting() throws Exception {
- Configuration conf = new Configuration(false);
- UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
-
- AppContext appContext = mock(AppContext.class);
- TaskAttemptListener tal = mock(TaskAttemptListener.class);
-
- String customLauncherName = "customLauncher";
- List<NamedEntityDescriptor> launcherDescriptors = new LinkedList<>();
- ByteBuffer bb = ByteBuffer.allocate(4);
- bb.putInt(0, 3);
- UserPayload customPayload = UserPayload.create(bb);
- launcherDescriptors.add(
- new NamedEntityDescriptor(customLauncherName, FakeContainerLauncher.class.getName())
- .setUserPayload(customPayload));
- launcherDescriptors
- .add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
- .setUserPayload(userPayload));
-
- ContainerLaucherRouterForMultipleLauncherTest clr =
- new ContainerLaucherRouterForMultipleLauncherTest(appContext, tal, null,
- launcherDescriptors,
- true);
- try {
- clr.init(conf);
- clr.start();
-
- assertEquals(2, clr.getNumContainerLaunchers());
- assertTrue(clr.getYarnContainerLauncherCreated());
- assertFalse(clr.getUberContainerLauncherCreated());
- assertEquals(customLauncherName, clr.getContainerLauncherName(0));
- assertEquals(TezConstants.getTezYarnServicePluginName(), clr.getContainerLauncherName(1));
-
- verify(clr.getTestContainerLauncher(0)).initialize();
- verify(clr.getTestContainerLauncher(0)).start();
- verify(clr.getTestContainerLauncher(1)).initialize();
- verify(clr.getTestContainerLauncher(1)).start();
-
- ContainerLaunchContext clc1 = mock(ContainerLaunchContext.class);
- Container container1 = mock(Container.class);
-
- ContainerLaunchContext clc2 = mock(ContainerLaunchContext.class);
- Container container2 = mock(Container.class);
-
- NMCommunicatorLaunchRequestEvent launchRequestEvent1 =
- new NMCommunicatorLaunchRequestEvent(clc1, container1, 0, 0, 0);
- NMCommunicatorLaunchRequestEvent launchRequestEvent2 =
- new NMCommunicatorLaunchRequestEvent(clc2, container2, 1, 0, 0);
-
- clr.handle(launchRequestEvent1);
-
-
- ArgumentCaptor<ContainerLaunchRequest> captor =
- ArgumentCaptor.forClass(ContainerLaunchRequest.class);
- verify(clr.getTestContainerLauncher(0)).launchContainer(captor.capture());
- assertEquals(1, captor.getAllValues().size());
- ContainerLaunchRequest launchRequest1 = captor.getValue();
- assertEquals(clc1, launchRequest1.getContainerLaunchContext());
-
- clr.handle(launchRequestEvent2);
- captor = ArgumentCaptor.forClass(ContainerLaunchRequest.class);
- verify(clr.getTestContainerLauncher(1)).launchContainer(captor.capture());
- assertEquals(1, captor.getAllValues().size());
- ContainerLaunchRequest launchRequest2 = captor.getValue();
- assertEquals(clc2, launchRequest2.getContainerLaunchContext());
-
- } finally {
- clr.stop();
- verify(clr.getTestContainerLauncher(0)).shutdown();
- verify(clr.getTestContainerLauncher(1)).shutdown();
- }
- }
-
- private static class ContainerLaucherRouterForMultipleLauncherTest
- extends ContainerLauncherRouter {
-
- // All variables setup as static since methods being overridden are invoked by the ContainerLauncherRouter ctor,
- // and regular variables will not be initialized at this point.
- private static final AtomicInteger numContainerLaunchers = new AtomicInteger(0);
- private static final Set<Integer> containerLauncherIndices = new HashSet<>();
- private static final ContainerLauncher yarnContainerLauncher = mock(ContainerLauncher.class);
- private static final ContainerLauncher uberContainerlauncher = mock(ContainerLauncher.class);
- private static final AtomicBoolean yarnContainerLauncherCreated = new AtomicBoolean(false);
- private static final AtomicBoolean uberContainerLauncherCreated = new AtomicBoolean(false);
-
- private static final List<ContainerLauncherContext> containerLauncherContexts =
- new LinkedList<>();
- private static final List<String> containerLauncherNames = new LinkedList<>();
- private static final List<ContainerLauncher> testContainerLaunchers = new LinkedList<>();
-
-
- public static void reset() {
- numContainerLaunchers.set(0);
- containerLauncherIndices.clear();
- yarnContainerLauncherCreated.set(false);
- uberContainerLauncherCreated.set(false);
- containerLauncherContexts.clear();
- containerLauncherNames.clear();
- testContainerLaunchers.clear();
- }
-
- public ContainerLaucherRouterForMultipleLauncherTest(AppContext context,
- TaskAttemptListener taskAttemptListener,
- String workingDirectory,
- List<NamedEntityDescriptor> containerLauncherDescriptors,
- boolean isPureLocalMode) throws
- UnknownHostException {
- super(context, taskAttemptListener, workingDirectory,
- containerLauncherDescriptors, isPureLocalMode);
- }
-
- @Override
- ContainerLauncher createContainerLauncher(NamedEntityDescriptor containerLauncherDescriptor,
- AppContext context,
- ContainerLauncherContext containerLauncherContext,
- TaskAttemptListener taskAttemptListener,
- String workingDirectory,
- int containerLauncherIndex,
- boolean isPureLocalMode) {
- numContainerLaunchers.incrementAndGet();
- boolean added = containerLauncherIndices.add(containerLauncherIndex);
- assertTrue("Cannot add multiple launchers with the same index", added);
- containerLauncherNames.add(containerLauncherDescriptor.getEntityName());
- containerLauncherContexts.add(containerLauncherContext);
- return super
- .createContainerLauncher(containerLauncherDescriptor, context, containerLauncherContext,
- taskAttemptListener, workingDirectory, containerLauncherIndex, isPureLocalMode);
- }
-
- @Override
- ContainerLauncher createYarnContainerLauncher(
- ContainerLauncherContext containerLauncherContext) {
- yarnContainerLauncherCreated.set(true);
- testContainerLaunchers.add(yarnContainerLauncher);
- return yarnContainerLauncher;
- }
-
- @Override
- ContainerLauncher createUberContainerLauncher(ContainerLauncherContext containerLauncherContext,
- AppContext context,
- TaskAttemptListener taskAttemptListener,
- String workingDirectory,
- boolean isPureLocalMode) {
- uberContainerLauncherCreated.set(true);
- testContainerLaunchers.add(uberContainerlauncher);
- return uberContainerlauncher;
- }
-
- @Override
- ContainerLauncher createCustomContainerLauncher(
- ContainerLauncherContext containerLauncherContext,
- NamedEntityDescriptor containerLauncherDescriptor) {
- ContainerLauncher spyLauncher = spy(super.createCustomContainerLauncher(
- containerLauncherContext, containerLauncherDescriptor));
- testContainerLaunchers.add(spyLauncher);
- return spyLauncher;
- }
-
- public int getNumContainerLaunchers() {
- return numContainerLaunchers.get();
- }
-
- public boolean getYarnContainerLauncherCreated() {
- return yarnContainerLauncherCreated.get();
- }
-
- public boolean getUberContainerLauncherCreated() {
- return uberContainerLauncherCreated.get();
- }
-
- public String getContainerLauncherName(int containerLauncherIndex) {
- return containerLauncherNames.get(containerLauncherIndex);
- }
-
- public ContainerLauncher getTestContainerLauncher(int containerLauncherIndex) {
- return testContainerLaunchers.get(containerLauncherIndex);
- }
-
- public ContainerLauncherContext getContainerLauncherContext(int containerLauncherIndex) {
- return containerLauncherContexts.get(containerLauncherIndex);
- }
- }
-
- private static class FakeContainerLauncher extends ContainerLauncher {
-
- public FakeContainerLauncher(
- ContainerLauncherContext containerLauncherContext) {
- super(containerLauncherContext);
- }
-
- @Override
- public void launchContainer(ContainerLaunchRequest launchRequest) {
-
- }
-
- @Override
- public void stopContainer(ContainerStopRequest stopRequest) {
-
- }
- }
-
-}
[4/7] tez git commit: TEZ-2708. Rename classes and variables post
TEZ-2003 changes. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
deleted file mode 100644
index 5159aff..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag.java
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.common.ContainerContext;
-import org.apache.tez.common.ContainerTask;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.common.security.JobTokenIdentifier;
-import org.apache.tez.common.security.JobTokenSecretManager;
-import org.apache.tez.common.security.TokenCache;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TaskCommunicator;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.serviceplugins.api.ContainerEndReason;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.dag.api.TaskHeartbeatRequest;
-import org.apache.tez.dag.api.TaskHeartbeatResponse;
-import org.apache.tez.dag.api.TezException;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.tez.common.TezTaskUmbilicalProtocol;
-import org.apache.tez.dag.api.TaskCommunicatorContext;
-import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.app.dag.Vertex;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
-import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
-import org.apache.tez.dag.app.dag.event.VertexEventType;
-import org.apache.tez.dag.app.rm.container.AMContainer;
-import org.apache.tez.dag.app.rm.container.AMContainerMap;
-import org.apache.tez.dag.app.rm.container.AMContainerTask;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-import org.apache.tez.dag.records.TezVertexID;
-import org.apache.tez.runtime.api.events.DataMovementEvent;
-import org.apache.tez.runtime.api.events.TaskAttemptCompletedEvent;
-import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent;
-import org.apache.tez.runtime.api.impl.EventType;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.runtime.api.impl.TezEvent;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-
-@SuppressWarnings("unchecked")
-// TODO TEZ-2003 (post) TEZ-2696 Rename to TestTezTaskCommunicator
-public class TestTaskAttemptListenerImplTezDag {
- private ApplicationId appId;
- private ApplicationAttemptId appAttemptId;
- private AppContext appContext;
- Credentials credentials;
- AMContainerMap amContainerMap;
- EventHandler eventHandler;
- DAG dag;
- TaskAttemptListenerImpTezDag taskAttemptListener;
- ContainerTask containerTask;
- AMContainerTask amContainerTask;
- TaskSpec taskSpec;
-
- TezVertexID vertexID;
- TezTaskID taskID;
- TezTaskAttemptID taskAttemptID;
-
- @Before
- public void setUp() {
- appId = ApplicationId.newInstance(1000, 1);
- appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
- dag = mock(DAG.class);
- TezDAGID dagID = TezDAGID.getInstance(appId, 1);
- vertexID = TezVertexID.getInstance(dagID, 1);
- taskID = TezTaskID.getInstance(vertexID, 1);
- taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
- credentials = new Credentials();
-
- amContainerMap = mock(AMContainerMap.class);
- Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
-
- eventHandler = mock(EventHandler.class);
-
- MockClock clock = new MockClock();
-
- appContext = mock(AppContext.class);
- doReturn(eventHandler).when(appContext).getEventHandler();
- doReturn(dag).when(appContext).getCurrentDAG();
- doReturn(appAcls).when(appContext).getApplicationACLs();
- doReturn(amContainerMap).when(appContext).getAllContainers();
- doReturn(clock).when(appContext).getClock();
-
- doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
- doReturn(credentials).when(appContext).getAppCredentials();
- NodeId nodeId = NodeId.newInstance("localhost", 0);
- AMContainer amContainer = mock(AMContainer.class);
- Container container = mock(Container.class);
- doReturn(nodeId).when(container).getNodeId();
- doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
- doReturn(container).when(amContainer).getContainer();
-
- Configuration conf = new TezConfiguration();
- UserPayload defaultPayload;
- try {
- defaultPayload = TezUtils.createUserPayloadFromConf(conf);
- } catch (IOException e) {
- throw new TezUncheckedException(e);
- }
- taskAttemptListener = new TaskAttemptListenerImplForTest(appContext,
- mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class),
- Lists.newArrayList(
- new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
- .setUserPayload(defaultPayload)));
-
- taskSpec = mock(TaskSpec.class);
- doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();
- amContainerTask = new AMContainerTask(taskSpec, null, null, false, 0);
- containerTask = null;
- }
-
- @Test(timeout = 5000)
- public void testGetTask() throws IOException {
-
- TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
- TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
-
- ContainerId containerId1 = createContainerId(appId, 1);
- ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
- containerTask = tezUmbilical.getTask(containerContext1);
- assertTrue(containerTask.shouldDie());
-
- ContainerId containerId2 = createContainerId(appId, 2);
- ContainerContext containerContext2 = new ContainerContext(containerId2.toString());
- taskAttemptListener.registerRunningContainer(containerId2, 0);
- containerTask = tezUmbilical.getTask(containerContext2);
- assertNull(containerTask);
-
- // Valid task registered
- taskAttemptListener.registerTaskAttempt(amContainerTask, containerId2, 0);
- containerTask = tezUmbilical.getTask(containerContext2);
- assertFalse(containerTask.shouldDie());
- assertEquals(taskSpec, containerTask.getTaskSpec());
-
- // Task unregistered. Should respond to heartbeats
- taskAttemptListener.unregisterTaskAttempt(taskAttemptID, 0, TaskAttemptEndReason.OTHER, null);
- containerTask = tezUmbilical.getTask(containerContext2);
- assertNull(containerTask);
-
- // Container unregistered. Should send a shouldDie = true
- taskAttemptListener.unregisterRunningContainer(containerId2, 0, ContainerEndReason.OTHER, null);
- containerTask = tezUmbilical.getTask(containerContext2);
- assertTrue(containerTask.shouldDie());
-
- ContainerId containerId3 = createContainerId(appId, 3);
- ContainerContext containerContext3 = new ContainerContext(containerId3.toString());
- taskAttemptListener.registerRunningContainer(containerId3, 0);
-
- // Register task to container3, followed by unregistering container 3 all together
- TaskSpec taskSpec2 = mock(TaskSpec.class);
- TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
- doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
- AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec, null, null, false, 0);
- taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId3, 0);
- taskAttemptListener.unregisterRunningContainer(containerId3, 0, ContainerEndReason.OTHER, null);
- containerTask = tezUmbilical.getTask(containerContext3);
- assertTrue(containerTask.shouldDie());
- }
-
- @Test(timeout = 5000)
- public void testGetTaskMultiplePulls() throws IOException {
- TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
- TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
-
- ContainerId containerId1 = createContainerId(appId, 1);
-
- ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
- taskAttemptListener.registerRunningContainer(containerId1, 0);
- containerTask = tezUmbilical.getTask(containerContext1);
- assertNull(containerTask);
-
- // Register task
- taskAttemptListener.registerTaskAttempt(amContainerTask, containerId1, 0);
- containerTask = tezUmbilical.getTask(containerContext1);
- assertFalse(containerTask.shouldDie());
- assertEquals(taskSpec, containerTask.getTaskSpec());
-
- // Try pulling again - simulates re-use pull
- containerTask = tezUmbilical.getTask(containerContext1);
- assertNull(containerTask);
- }
-
- @Test (timeout = 5000)
- public void testTaskEventRouting() throws Exception {
- List<TezEvent> events = Arrays.asList(
- new TezEvent(new TaskStatusUpdateEvent(null, 0.0f, null), null),
- new TezEvent(DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0])), null),
- new TezEvent(new TaskAttemptCompletedEvent(), null)
- );
-
- generateHeartbeat(events, 0, 1, 0, new ArrayList<TezEvent>());
-
- ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
- verify(eventHandler, times(2)).handle(arg.capture());
- final List<Event> argAllValues = arg.getAllValues();
-
- final Event statusUpdateEvent = argAllValues.get(0);
- assertEquals("First event should be status update", TaskAttemptEventType.TA_STATUS_UPDATE,
- statusUpdateEvent.getType());
-
- final Event vertexEvent = argAllValues.get(1);
- final VertexEventRouteEvent vertexRouteEvent = (VertexEventRouteEvent)vertexEvent;
- assertEquals("First event should be routed to vertex", VertexEventType.V_ROUTE_EVENT,
- vertexEvent.getType());
- assertEquals(EventType.DATA_MOVEMENT_EVENT,
- vertexRouteEvent.getEvents().get(0).getEventType());
- assertEquals(EventType.TASK_ATTEMPT_COMPLETED_EVENT,
- vertexRouteEvent.getEvents().get(1).getEventType());
-
- }
-
- @Test (timeout = 5000)
- public void testTaskEventRoutingTaskAttemptOnly() throws Exception {
- List<TezEvent> events = Arrays.asList(
- new TezEvent(new TaskAttemptCompletedEvent(), null)
- );
- generateHeartbeat(events, 0, 1, 0, new ArrayList<TezEvent>());
-
- ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
- verify(eventHandler, times(1)).handle(arg.capture());
- final List<Event> argAllValues = arg.getAllValues();
-
- final Event event = argAllValues.get(0);
- assertEquals("only event should be route event", VertexEventType.V_ROUTE_EVENT,
- event.getType());
- }
-
- @Test (timeout = 5000)
- public void testTaskHeartbeatResponse() throws Exception {
- List<TezEvent> events = new ArrayList<TezEvent>();
- List<TezEvent> eventsToSend = new ArrayList<TezEvent>();
- TaskHeartbeatResponse response = generateHeartbeat(events, 0, 1, 2, eventsToSend);
-
- assertEquals(2, response.getNextFromEventId());
- assertEquals(eventsToSend, response.getEvents());
- }
-
- //try 10 times to allocate random port, fail it if no one is succeed.
- @Test (timeout = 5000)
- public void testPortRange() {
- boolean succeedToAllocate = false;
- Random rand = new Random();
- for (int i = 0; i < 10; ++i) {
- int nextPort = 1024 + rand.nextInt(65535 - 1024);
- if (testPortRange(nextPort)) {
- succeedToAllocate = true;
- break;
- }
- }
- if (!succeedToAllocate) {
- fail("Can not allocate free port even in 10 iterations for TaskAttemptListener");
- }
- }
-
- // TODO TEZ-2003 Move this into TestTezTaskCommunicator. Potentially other tests as well.
- @Test (timeout= 5000)
- public void testPortRange_NotSpecified() throws IOException {
- Configuration conf = new Configuration();
- JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(
- "fakeIdentifier"));
- Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier,
- new JobTokenSecretManager());
- sessionToken.setService(identifier.getJobId());
- TokenCache.setSessionToken(sessionToken, credentials);
- UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
- taskAttemptListener = new TaskAttemptListenerImpTezDag(appContext,
- mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(
- new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
- .setUserPayload(userPayload)));
- // no exception happen, should started properly
- taskAttemptListener.init(conf);
- taskAttemptListener.start();
- }
-
- private boolean testPortRange(int port) {
- boolean succeedToAllocate = true;
- try {
- Configuration conf = new Configuration();
-
- JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(
- "fakeIdentifier"));
- Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier,
- new JobTokenSecretManager());
- sessionToken.setService(identifier.getJobId());
- TokenCache.setSessionToken(sessionToken, credentials);
-
- conf.set(TezConfiguration.TEZ_AM_TASK_AM_PORT_RANGE, port + "-" + port);
- UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
-
- taskAttemptListener = new TaskAttemptListenerImpTezDag(appContext,
- mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists
- .newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
- .setUserPayload(userPayload)));
- taskAttemptListener.init(conf);
- taskAttemptListener.start();
- int resultedPort = taskAttemptListener.getTaskCommunicator(0).getAddress().getPort();
- assertEquals(port, resultedPort);
- } catch (Exception e) {
- succeedToAllocate = false;
- } finally {
- if (taskAttemptListener != null) {
- try {
- taskAttemptListener.close();
- } catch (IOException e) {
- e.printStackTrace();
- fail("fail to stop TaskAttemptListener");
- }
- }
- }
- return succeedToAllocate;
- }
-
- private TaskHeartbeatResponse generateHeartbeat(List<TezEvent> events,
- int fromEventId, int maxEvents, int nextFromEventId,
- List<TezEvent> sendEvents) throws IOException, TezException {
- ContainerId containerId = createContainerId(appId, 1);
- Vertex vertex = mock(Vertex.class);
-
- doReturn(vertex).when(dag).getVertex(vertexID);
- doReturn("test_vertex").when(vertex).getName();
- TaskAttemptEventInfo eventInfo = new TaskAttemptEventInfo(nextFromEventId, sendEvents, 0);
- doReturn(eventInfo).when(vertex).getTaskAttemptTezEvents(taskAttemptID, fromEventId, 0, maxEvents);
-
- taskAttemptListener.registerRunningContainer(containerId, 0);
- taskAttemptListener.registerTaskAttempt(amContainerTask, containerId, 0);
-
- TaskHeartbeatRequest request = mock(TaskHeartbeatRequest.class);
- doReturn(containerId.toString()).when(request).getContainerIdentifier();
- doReturn(containerId.toString()).when(request).getContainerIdentifier();
- doReturn(taskAttemptID).when(request).getTaskAttemptId();
- doReturn(events).when(request).getEvents();
- doReturn(maxEvents).when(request).getMaxEvents();
- doReturn(fromEventId).when(request).getStartIndex();
-
- return taskAttemptListener.heartbeat(request);
- }
-
-
- private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
- ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
- return ContainerId.newInstance(appAttemptId, containerIdx);
- }
-
- private static class TaskAttemptListenerImplForTest extends TaskAttemptListenerImpTezDag {
-
- public TaskAttemptListenerImplForTest(AppContext context,
- TaskHeartbeatHandler thh,
- ContainerHeartbeatHandler chh,
- List<NamedEntityDescriptor> taskCommDescriptors) {
- super(context, thh, chh, taskCommDescriptors);
- }
-
- @Override
- TaskCommunicator createDefaultTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
- return new TezTaskCommunicatorImplForTest(taskCommunicatorContext);
- }
-
- }
-
- private static class TezTaskCommunicatorImplForTest extends TezTaskCommunicatorImpl {
-
- public TezTaskCommunicatorImplForTest(
- TaskCommunicatorContext taskCommunicatorContext) {
- super(taskCommunicatorContext);
- }
-
- @Override
- protected void startRpcServer() {
- }
-
- @Override
- protected void stopRpcServer() {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java
deleted file mode 100644
index 74468f2..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskAttemptListenerImplTezDag2.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptKilled;
-import org.apache.tez.dag.app.rm.container.AMContainer;
-import org.apache.tez.dag.app.rm.container.AMContainerMap;
-import org.apache.tez.dag.app.rm.container.AMContainerTask;
-import org.apache.tez.dag.records.TaskAttemptTerminationCause;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-
-// TODO TEZ-2003. Rename to TestTaskAttemptListener | whatever TaskAttemptListener is renamed to.
-public class TestTaskAttemptListenerImplTezDag2 {
-
- @Test(timeout = 5000)
- public void testTaskAttemptFailedKilled() throws IOException {
- ApplicationId appId = ApplicationId.newInstance(1000, 1);
- ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
- Credentials credentials = new Credentials();
- AppContext appContext = mock(AppContext.class);
- EventHandler eventHandler = mock(EventHandler.class);
- DAG dag = mock(DAG.class);
- AMContainerMap amContainerMap = mock(AMContainerMap.class);
- Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
- doReturn(eventHandler).when(appContext).getEventHandler();
- doReturn(dag).when(appContext).getCurrentDAG();
- doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
- doReturn(credentials).when(appContext).getAppCredentials();
- doReturn(appAcls).when(appContext).getApplicationACLs();
- doReturn(amContainerMap).when(appContext).getAllContainers();
- NodeId nodeId = NodeId.newInstance("localhost", 0);
- AMContainer amContainer = mock(AMContainer.class);
- Container container = mock(Container.class);
- doReturn(nodeId).when(container).getNodeId();
- doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
- doReturn(container).when(amContainer).getContainer();
-
- Configuration conf = new TezConfiguration();
- UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
- TaskAttemptListenerImpTezDag taskAttemptListener =
- new TaskAttemptListenerImpTezDag(appContext, mock(TaskHeartbeatHandler.class),
- mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(
- TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload)));
-
- TaskSpec taskSpec1 = mock(TaskSpec.class);
- TezTaskAttemptID taskAttemptId1 = mock(TezTaskAttemptID.class);
- doReturn(taskAttemptId1).when(taskSpec1).getTaskAttemptID();
- AMContainerTask amContainerTask1 = new AMContainerTask(taskSpec1, null, null, false, 10);
-
- TaskSpec taskSpec2 = mock(TaskSpec.class);
- TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
- doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
- AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec2, null, null, false, 10);
-
- ContainerId containerId1 = createContainerId(appId, 1);
- taskAttemptListener.registerRunningContainer(containerId1, 0);
- taskAttemptListener.registerTaskAttempt(amContainerTask1, containerId1, 0);
- ContainerId containerId2 = createContainerId(appId, 2);
- taskAttemptListener.registerRunningContainer(containerId2, 0);
- taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId2, 0);
-
-
- taskAttemptListener
- .taskFailed(taskAttemptId1, TaskAttemptEndReason.COMMUNICATION_ERROR, "Diagnostics1");
- taskAttemptListener
- .taskKilled(taskAttemptId2, TaskAttemptEndReason.EXECUTOR_BUSY, "Diagnostics2");
-
- ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
- verify(eventHandler, times(2)).handle(argumentCaptor.capture());
- assertTrue(argumentCaptor.getAllValues().get(0) instanceof TaskAttemptEventAttemptFailed);
- assertTrue(argumentCaptor.getAllValues().get(1) instanceof TaskAttemptEventAttemptKilled);
- TaskAttemptEventAttemptFailed failedEvent =
- (TaskAttemptEventAttemptFailed) argumentCaptor.getAllValues().get(0);
- TaskAttemptEventAttemptKilled killedEvent =
- (TaskAttemptEventAttemptKilled) argumentCaptor.getAllValues().get(1);
-
- assertEquals("Diagnostics1", failedEvent.getDiagnosticInfo());
- assertEquals(TaskAttemptTerminationCause.COMMUNICATION_ERROR,
- failedEvent.getTerminationCause());
-
- assertEquals("Diagnostics2", killedEvent.getDiagnosticInfo());
- assertEquals(TaskAttemptTerminationCause.SERVICE_BUSY, killedEvent.getTerminationCause());
- // TODO TEZ-2003. Verify unregistration from the registered list
- }
-
- private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
- ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
- ContainerId containerId = ContainerId.newInstance(appAttemptId, containerIdx);
- return containerId;
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorContextImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorContextImpl.java
index 1545eb4..5222a2d 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorContextImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorContextImpl.java
@@ -34,7 +34,7 @@ public class TestTaskCommunicatorContextImpl {
@Test(timeout = 5000)
public void testIsKnownContainer() {
AppContext appContext = mock(AppContext.class);
- TaskAttemptListenerImpTezDag tal = mock(TaskAttemptListenerImpTezDag.class);
+ TaskCommunicatorManager tal = mock(TaskCommunicatorManager.class);
AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class), tal, mock(
ContainerSignatureMatcher.class), appContext);
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java
index 4f68fab..be7adde 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java
@@ -220,7 +220,7 @@ public class TestTaskCommunicatorManager {
}
- static class TaskCommManagerForMultipleCommTest extends TaskAttemptListenerImpTezDag {
+ static class TaskCommManagerForMultipleCommTest extends TaskCommunicatorManager {
// All variables setup as static since methods being overridden are invoked by the ContainerLauncherRouter ctor,
// and regular variables will not be initialized at this point.
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java
new file mode 100644
index 0000000..e8ce429
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java
@@ -0,0 +1,425 @@
+/*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.app;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.ContainerContext;
+import org.apache.tez.common.ContainerTask;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TaskCommunicator;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.serviceplugins.api.ContainerEndReason;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.dag.api.TaskHeartbeatRequest;
+import org.apache.tez.dag.api.TaskHeartbeatResponse;
+import org.apache.tez.dag.api.TezException;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
+import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
+import org.apache.tez.dag.app.dag.event.VertexEventType;
+import org.apache.tez.dag.app.rm.container.AMContainer;
+import org.apache.tez.dag.app.rm.container.AMContainerMap;
+import org.apache.tez.dag.app.rm.container.AMContainerTask;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.api.events.DataMovementEvent;
+import org.apache.tez.runtime.api.events.TaskAttemptCompletedEvent;
+import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent;
+import org.apache.tez.runtime.api.impl.EventType;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.runtime.api.impl.TezEvent;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+@SuppressWarnings("unchecked")
+public class TestTaskCommunicatorManager1 {
+ private ApplicationId appId;
+ private ApplicationAttemptId appAttemptId;
+ private AppContext appContext;
+ Credentials credentials;
+ AMContainerMap amContainerMap;
+ EventHandler eventHandler;
+ DAG dag;
+ TaskCommunicatorManager taskAttemptListener;
+ ContainerTask containerTask;
+ AMContainerTask amContainerTask;
+ TaskSpec taskSpec;
+
+ TezVertexID vertexID;
+ TezTaskID taskID;
+ TezTaskAttemptID taskAttemptID;
+
+ @Before
+ public void setUp() {
+ appId = ApplicationId.newInstance(1000, 1);
+ appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+ dag = mock(DAG.class);
+ TezDAGID dagID = TezDAGID.getInstance(appId, 1);
+ vertexID = TezVertexID.getInstance(dagID, 1);
+ taskID = TezTaskID.getInstance(vertexID, 1);
+ taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
+ credentials = new Credentials();
+
+ amContainerMap = mock(AMContainerMap.class);
+ Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
+
+ eventHandler = mock(EventHandler.class);
+
+ MockClock clock = new MockClock();
+
+ appContext = mock(AppContext.class);
+ doReturn(eventHandler).when(appContext).getEventHandler();
+ doReturn(dag).when(appContext).getCurrentDAG();
+ doReturn(appAcls).when(appContext).getApplicationACLs();
+ doReturn(amContainerMap).when(appContext).getAllContainers();
+ doReturn(clock).when(appContext).getClock();
+
+ doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
+ doReturn(credentials).when(appContext).getAppCredentials();
+ NodeId nodeId = NodeId.newInstance("localhost", 0);
+ AMContainer amContainer = mock(AMContainer.class);
+ Container container = mock(Container.class);
+ doReturn(nodeId).when(container).getNodeId();
+ doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
+ doReturn(container).when(amContainer).getContainer();
+
+ Configuration conf = new TezConfiguration();
+ UserPayload defaultPayload;
+ try {
+ defaultPayload = TezUtils.createUserPayloadFromConf(conf);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ taskAttemptListener = new TaskCommunicatorManagerInterfaceImplForTest(appContext,
+ mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class),
+ Lists.newArrayList(
+ new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+ .setUserPayload(defaultPayload)));
+
+ taskSpec = mock(TaskSpec.class);
+ doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();
+ amContainerTask = new AMContainerTask(taskSpec, null, null, false, 0);
+ containerTask = null;
+ }
+
+ @Test(timeout = 5000)
+ public void testGetTask() throws IOException {
+
+ TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
+ TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
+
+ ContainerId containerId1 = createContainerId(appId, 1);
+ ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
+ containerTask = tezUmbilical.getTask(containerContext1);
+ assertTrue(containerTask.shouldDie());
+
+ ContainerId containerId2 = createContainerId(appId, 2);
+ ContainerContext containerContext2 = new ContainerContext(containerId2.toString());
+ taskAttemptListener.registerRunningContainer(containerId2, 0);
+ containerTask = tezUmbilical.getTask(containerContext2);
+ assertNull(containerTask);
+
+ // Valid task registered
+ taskAttemptListener.registerTaskAttempt(amContainerTask, containerId2, 0);
+ containerTask = tezUmbilical.getTask(containerContext2);
+ assertFalse(containerTask.shouldDie());
+ assertEquals(taskSpec, containerTask.getTaskSpec());
+
+ // Task unregistered. Should respond to heartbeats
+ taskAttemptListener.unregisterTaskAttempt(taskAttemptID, 0, TaskAttemptEndReason.OTHER, null);
+ containerTask = tezUmbilical.getTask(containerContext2);
+ assertNull(containerTask);
+
+ // Container unregistered. Should send a shouldDie = true
+ taskAttemptListener.unregisterRunningContainer(containerId2, 0, ContainerEndReason.OTHER, null);
+ containerTask = tezUmbilical.getTask(containerContext2);
+ assertTrue(containerTask.shouldDie());
+
+ ContainerId containerId3 = createContainerId(appId, 3);
+ ContainerContext containerContext3 = new ContainerContext(containerId3.toString());
+ taskAttemptListener.registerRunningContainer(containerId3, 0);
+
+ // Register task to container3, followed by unregistering container 3 all together
+ TaskSpec taskSpec2 = mock(TaskSpec.class);
+ TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
+ doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
+ AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec, null, null, false, 0);
+ taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId3, 0);
+ taskAttemptListener.unregisterRunningContainer(containerId3, 0, ContainerEndReason.OTHER, null);
+ containerTask = tezUmbilical.getTask(containerContext3);
+ assertTrue(containerTask.shouldDie());
+ }
+
+ @Test(timeout = 5000)
+ public void testGetTaskMultiplePulls() throws IOException {
+ TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl)taskAttemptListener.getTaskCommunicator(0);
+ TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
+
+ ContainerId containerId1 = createContainerId(appId, 1);
+
+ ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
+ taskAttemptListener.registerRunningContainer(containerId1, 0);
+ containerTask = tezUmbilical.getTask(containerContext1);
+ assertNull(containerTask);
+
+ // Register task
+ taskAttemptListener.registerTaskAttempt(amContainerTask, containerId1, 0);
+ containerTask = tezUmbilical.getTask(containerContext1);
+ assertFalse(containerTask.shouldDie());
+ assertEquals(taskSpec, containerTask.getTaskSpec());
+
+ // Try pulling again - simulates re-use pull
+ containerTask = tezUmbilical.getTask(containerContext1);
+ assertNull(containerTask);
+ }
+
+ @Test (timeout = 5000)
+ public void testTaskEventRouting() throws Exception {
+ List<TezEvent> events = Arrays.asList(
+ new TezEvent(new TaskStatusUpdateEvent(null, 0.0f, null), null),
+ new TezEvent(DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0])), null),
+ new TezEvent(new TaskAttemptCompletedEvent(), null)
+ );
+
+ generateHeartbeat(events, 0, 1, 0, new ArrayList<TezEvent>());
+
+ ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(2)).handle(arg.capture());
+ final List<Event> argAllValues = arg.getAllValues();
+
+ final Event statusUpdateEvent = argAllValues.get(0);
+ assertEquals("First event should be status update", TaskAttemptEventType.TA_STATUS_UPDATE,
+ statusUpdateEvent.getType());
+
+ final Event vertexEvent = argAllValues.get(1);
+ final VertexEventRouteEvent vertexRouteEvent = (VertexEventRouteEvent)vertexEvent;
+ assertEquals("First event should be routed to vertex", VertexEventType.V_ROUTE_EVENT,
+ vertexEvent.getType());
+ assertEquals(EventType.DATA_MOVEMENT_EVENT,
+ vertexRouteEvent.getEvents().get(0).getEventType());
+ assertEquals(EventType.TASK_ATTEMPT_COMPLETED_EVENT,
+ vertexRouteEvent.getEvents().get(1).getEventType());
+
+ }
+
+ @Test (timeout = 5000)
+ public void testTaskEventRoutingTaskAttemptOnly() throws Exception {
+ List<TezEvent> events = Arrays.asList(
+ new TezEvent(new TaskAttemptCompletedEvent(), null)
+ );
+ generateHeartbeat(events, 0, 1, 0, new ArrayList<TezEvent>());
+
+ ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(1)).handle(arg.capture());
+ final List<Event> argAllValues = arg.getAllValues();
+
+ final Event event = argAllValues.get(0);
+ assertEquals("only event should be route event", VertexEventType.V_ROUTE_EVENT,
+ event.getType());
+ }
+
+ @Test (timeout = 5000)
+ public void testTaskHeartbeatResponse() throws Exception {
+ List<TezEvent> events = new ArrayList<TezEvent>();
+ List<TezEvent> eventsToSend = new ArrayList<TezEvent>();
+ TaskHeartbeatResponse response = generateHeartbeat(events, 0, 1, 2, eventsToSend);
+
+ assertEquals(2, response.getNextFromEventId());
+ assertEquals(eventsToSend, response.getEvents());
+ }
+
+ //try 10 times to allocate random port, fail it if no one is succeed.
+ @Test (timeout = 5000)
+ public void testPortRange() {
+ boolean succeedToAllocate = false;
+ Random rand = new Random();
+ for (int i = 0; i < 10; ++i) {
+ int nextPort = 1024 + rand.nextInt(65535 - 1024);
+ if (testPortRange(nextPort)) {
+ succeedToAllocate = true;
+ break;
+ }
+ }
+ if (!succeedToAllocate) {
+ fail("Can not allocate free port even in 10 iterations for TaskAttemptListener");
+ }
+ }
+
+ // TODO TEZ-2003 Move this into TestTezTaskCommunicator. Potentially other tests as well.
+ @Test (timeout= 5000)
+ public void testPortRange_NotSpecified() throws IOException {
+ Configuration conf = new Configuration();
+ JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(
+ "fakeIdentifier"));
+ Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier,
+ new JobTokenSecretManager());
+ sessionToken.setService(identifier.getJobId());
+ TokenCache.setSessionToken(sessionToken, credentials);
+ UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+ taskAttemptListener = new TaskCommunicatorManager(appContext,
+ mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(
+ new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+ .setUserPayload(userPayload)));
+ // no exception happen, should started properly
+ taskAttemptListener.init(conf);
+ taskAttemptListener.start();
+ }
+
+ private boolean testPortRange(int port) {
+ boolean succeedToAllocate = true;
+ try {
+ Configuration conf = new Configuration();
+
+ JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(
+ "fakeIdentifier"));
+ Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier,
+ new JobTokenSecretManager());
+ sessionToken.setService(identifier.getJobId());
+ TokenCache.setSessionToken(sessionToken, credentials);
+
+ conf.set(TezConfiguration.TEZ_AM_TASK_AM_PORT_RANGE, port + "-" + port);
+ UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+
+ taskAttemptListener = new TaskCommunicatorManager(appContext,
+ mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists
+ .newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
+ .setUserPayload(userPayload)));
+ taskAttemptListener.init(conf);
+ taskAttemptListener.start();
+ int resultedPort = taskAttemptListener.getTaskCommunicator(0).getAddress().getPort();
+ assertEquals(port, resultedPort);
+ } catch (Exception e) {
+ succeedToAllocate = false;
+ } finally {
+ if (taskAttemptListener != null) {
+ try {
+ taskAttemptListener.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail("fail to stop TaskAttemptListener");
+ }
+ }
+ }
+ return succeedToAllocate;
+ }
+
+ private TaskHeartbeatResponse generateHeartbeat(List<TezEvent> events,
+ int fromEventId, int maxEvents, int nextFromEventId,
+ List<TezEvent> sendEvents) throws IOException, TezException {
+ ContainerId containerId = createContainerId(appId, 1);
+ Vertex vertex = mock(Vertex.class);
+
+ doReturn(vertex).when(dag).getVertex(vertexID);
+ doReturn("test_vertex").when(vertex).getName();
+ TaskAttemptEventInfo eventInfo = new TaskAttemptEventInfo(nextFromEventId, sendEvents, 0);
+ doReturn(eventInfo).when(vertex).getTaskAttemptTezEvents(taskAttemptID, fromEventId, 0, maxEvents);
+
+ taskAttemptListener.registerRunningContainer(containerId, 0);
+ taskAttemptListener.registerTaskAttempt(amContainerTask, containerId, 0);
+
+ TaskHeartbeatRequest request = mock(TaskHeartbeatRequest.class);
+ doReturn(containerId.toString()).when(request).getContainerIdentifier();
+ doReturn(containerId.toString()).when(request).getContainerIdentifier();
+ doReturn(taskAttemptID).when(request).getTaskAttemptId();
+ doReturn(events).when(request).getEvents();
+ doReturn(maxEvents).when(request).getMaxEvents();
+ doReturn(fromEventId).when(request).getStartIndex();
+
+ return taskAttemptListener.heartbeat(request);
+ }
+
+
+ private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
+ return ContainerId.newInstance(appAttemptId, containerIdx);
+ }
+
+ private static class TaskCommunicatorManagerInterfaceImplForTest extends TaskCommunicatorManager {
+
+ public TaskCommunicatorManagerInterfaceImplForTest(AppContext context,
+ TaskHeartbeatHandler thh,
+ ContainerHeartbeatHandler chh,
+ List<NamedEntityDescriptor> taskCommDescriptors) {
+ super(context, thh, chh, taskCommDescriptors);
+ }
+
+ @Override
+ TaskCommunicator createDefaultTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
+ return new TezTaskCommunicatorImplForTest(taskCommunicatorContext);
+ }
+
+ }
+
+ private static class TezTaskCommunicatorImplForTest extends TezTaskCommunicatorImpl {
+
+ public TezTaskCommunicatorImplForTest(
+ TaskCommunicatorContext taskCommunicatorContext) {
+ super(taskCommunicatorContext);
+ }
+
+ @Override
+ protected void startRpcServer() {
+ }
+
+ @Override
+ protected void stopRpcServer() {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java
new file mode 100644
index 0000000..d75b0e5
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptKilled;
+import org.apache.tez.dag.app.rm.container.AMContainer;
+import org.apache.tez.dag.app.rm.container.AMContainerMap;
+import org.apache.tez.dag.app.rm.container.AMContainerTask;
+import org.apache.tez.dag.records.TaskAttemptTerminationCause;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+public class TestTaskCommunicatorManager2 {
+
+ @Test(timeout = 5000)
+ public void testTaskAttemptFailedKilled() throws IOException {
+ ApplicationId appId = ApplicationId.newInstance(1000, 1);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+ Credentials credentials = new Credentials();
+ AppContext appContext = mock(AppContext.class);
+ EventHandler eventHandler = mock(EventHandler.class);
+ DAG dag = mock(DAG.class);
+ AMContainerMap amContainerMap = mock(AMContainerMap.class);
+ Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
+ doReturn(eventHandler).when(appContext).getEventHandler();
+ doReturn(dag).when(appContext).getCurrentDAG();
+ doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
+ doReturn(credentials).when(appContext).getAppCredentials();
+ doReturn(appAcls).when(appContext).getApplicationACLs();
+ doReturn(amContainerMap).when(appContext).getAllContainers();
+ NodeId nodeId = NodeId.newInstance("localhost", 0);
+ AMContainer amContainer = mock(AMContainer.class);
+ Container container = mock(Container.class);
+ doReturn(nodeId).when(container).getNodeId();
+ doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
+ doReturn(container).when(amContainer).getContainer();
+
+ Configuration conf = new TezConfiguration();
+ UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+ TaskCommunicatorManager taskAttemptListener =
+ new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class),
+ mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(
+ TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload)));
+
+ TaskSpec taskSpec1 = mock(TaskSpec.class);
+ TezTaskAttemptID taskAttemptId1 = mock(TezTaskAttemptID.class);
+ doReturn(taskAttemptId1).when(taskSpec1).getTaskAttemptID();
+ AMContainerTask amContainerTask1 = new AMContainerTask(taskSpec1, null, null, false, 10);
+
+ TaskSpec taskSpec2 = mock(TaskSpec.class);
+ TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
+ doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
+ AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec2, null, null, false, 10);
+
+ ContainerId containerId1 = createContainerId(appId, 1);
+ taskAttemptListener.registerRunningContainer(containerId1, 0);
+ taskAttemptListener.registerTaskAttempt(amContainerTask1, containerId1, 0);
+ ContainerId containerId2 = createContainerId(appId, 2);
+ taskAttemptListener.registerRunningContainer(containerId2, 0);
+ taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId2, 0);
+
+
+ taskAttemptListener
+ .taskFailed(taskAttemptId1, TaskAttemptEndReason.COMMUNICATION_ERROR, "Diagnostics1");
+ taskAttemptListener
+ .taskKilled(taskAttemptId2, TaskAttemptEndReason.EXECUTOR_BUSY, "Diagnostics2");
+
+ ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(2)).handle(argumentCaptor.capture());
+ assertTrue(argumentCaptor.getAllValues().get(0) instanceof TaskAttemptEventAttemptFailed);
+ assertTrue(argumentCaptor.getAllValues().get(1) instanceof TaskAttemptEventAttemptKilled);
+ TaskAttemptEventAttemptFailed failedEvent =
+ (TaskAttemptEventAttemptFailed) argumentCaptor.getAllValues().get(0);
+ TaskAttemptEventAttemptKilled killedEvent =
+ (TaskAttemptEventAttemptKilled) argumentCaptor.getAllValues().get(1);
+
+ assertEquals("Diagnostics1", failedEvent.getDiagnosticInfo());
+ assertEquals(TaskAttemptTerminationCause.COMMUNICATION_ERROR,
+ failedEvent.getTerminationCause());
+
+ assertEquals("Diagnostics2", killedEvent.getDiagnosticInfo());
+ assertEquals(TaskAttemptTerminationCause.SERVICE_BUSY, killedEvent.getTerminationCause());
+ // TODO TEZ-2003. Verify unregistration from the registered list
+ }
+
+ private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) {
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
+ ContainerId containerId = ContainerId.newInstance(appAttemptId, containerIdx);
+ return containerId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java
index 83421a2..f0b89c8 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java
@@ -70,7 +70,7 @@ import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ClusterInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.DAGState;
import org.apache.tez.dag.app.dag.DAGTerminationCause;
@@ -142,7 +142,7 @@ public class TestCommit {
private TaskEventDispatcher taskEventDispatcher;
private VertexEventDispatcher vertexEventDispatcher;
private DagEventDispatcher dagEventDispatcher;
- private TaskAttemptListener taskAttemptListener;
+ private TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
private TaskHeartbeatHandler thh;
private Clock clock = new SystemClock();
private DAGFinishEventHandler dagFinishEventHandler;
@@ -317,7 +317,7 @@ public class TestCommit {
doReturn(historyEventHandler).when(appContext).getHistoryHandler();
doReturn(aclManager).when(appContext).getAMACLManager();
dag = new DAGImpl(dagId, conf, dagPlan, dispatcher.getEventHandler(),
- taskAttemptListener, fsTokens, clock, "user", thh, appContext);
+ taskCommunicatorManagerInterface, fsTokens, clock, "user", thh, appContext);
doReturn(dag).when(appContext).getCurrentDAG();
doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();
ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(8192,10));
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index e268a99..ac4f61b 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -88,7 +88,7 @@ import org.apache.tez.dag.api.records.DAGProtos.TezEntityDescriptorProto;
import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ClusterInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.DAGScheduler;
import org.apache.tez.dag.app.dag.DAGState;
@@ -165,7 +165,7 @@ public class TestDAGImpl {
private TaskEventDispatcher taskEventDispatcher;
private VertexEventDispatcher vertexEventDispatcher;
private DagEventDispatcher dagEventDispatcher;
- private TaskAttemptListener taskAttemptListener;
+ private TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
private TaskHeartbeatHandler thh;
private Clock clock = new SystemClock();
private DAGFinishEventHandler dagFinishEventHandler;
@@ -784,7 +784,7 @@ public class TestDAGImpl {
doReturn(historyEventHandler).when(appContext).getHistoryHandler();
doReturn(aclManager).when(appContext).getAMACLManager();
dag = new DAGImpl(dagId, conf, dagPlan,
- dispatcher.getEventHandler(), taskAttemptListener,
+ dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
fsTokens, clock, "user", thh, appContext);
dag.entityUpdateTracker = new StateChangeNotifierForTest(dag);
doReturn(dag).when(appContext).getCurrentDAG();
@@ -795,7 +795,7 @@ public class TestDAGImpl {
mrrDagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 2);
mrrDagPlan = createTestMRRDAGPlan();
mrrDag = new DAGImpl(mrrDagId, conf, mrrDagPlan,
- dispatcher.getEventHandler(), taskAttemptListener,
+ dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
fsTokens, clock, "user", thh,
mrrAppContext);
mrrDag.entityUpdateTracker = new StateChangeNotifierForTest(mrrDag);
@@ -811,7 +811,7 @@ public class TestDAGImpl {
groupDagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 3);
groupDagPlan = createGroupDAGPlan();
groupDag = new DAGImpl(groupDagId, conf, groupDagPlan,
- dispatcher.getEventHandler(), taskAttemptListener,
+ dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
fsTokens, clock, "user", thh,
groupAppContext);
groupDag.entityUpdateTracker = new StateChangeNotifierForTest(groupDag);
@@ -881,7 +881,7 @@ public class TestDAGImpl {
dagWithCustomEdgeAppContext = mock(AppContext.class);
doReturn(aclManager).when(dagWithCustomEdgeAppContext).getAMACLManager();
dagWithCustomEdge = new DAGImpl(dagWithCustomEdgeId, conf, dagPlanWithCustomEdge,
- dispatcher.getEventHandler(), taskAttemptListener,
+ dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
fsTokens, clock, "user", thh, dagWithCustomEdgeAppContext);
dagWithCustomEdge.entityUpdateTracker = new StateChangeNotifierForTest(dagWithCustomEdge);
doReturn(conf).when(dagWithCustomEdgeAppContext).getAMConf();
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
index 792fa63..409c506 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
@@ -37,7 +37,7 @@ import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ClusterInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.DAGState;
import org.apache.tez.dag.app.dag.DAGTerminationCause;
@@ -91,7 +91,7 @@ public class TestDAGRecovery {
DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
dag =
new DAGImpl(dagId, new Configuration(), dagPlan, mockEventHandler,
- mock(TaskAttemptListener.class), new Credentials(),
+ mock(TaskCommunicatorManagerInterface.class), new Credentials(),
new SystemClock(), user, mock(TaskHeartbeatHandler.class),
mockAppContext);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index 04bb2df..13c9202 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -64,7 +64,7 @@ import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ClusterInfo;
import org.apache.tez.dag.app.ContainerContext;
import org.apache.tez.dag.app.ContainerHeartbeatHandler;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttemptStateInternal;
@@ -145,7 +145,7 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(
TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1);
TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
- mock(TaskAttemptListener.class), new Configuration(), new SystemClock(),
+ mock(TaskCommunicatorManagerInterface.class), new Configuration(), new SystemClock(),
mock(TaskHeartbeatHandler.class), mock(AppContext.class),
false, Resource.newInstance(1024, 1), createFakeContainerContext(), false);
@@ -180,12 +180,12 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(
TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1);
TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
- mock(TaskAttemptListener.class), new Configuration(), new SystemClock(),
+ mock(TaskCommunicatorManagerInterface.class), new Configuration(), new SystemClock(),
mock(TaskHeartbeatHandler.class), mock(AppContext.class),
false, Resource.newInstance(1024, 1), createFakeContainerContext(), false);
TaskAttemptImpl taImplReScheduled = new MockTaskAttemptImpl(taskID, 1, eventHandler,
- mock(TaskAttemptListener.class), new Configuration(), new SystemClock(),
+ mock(TaskCommunicatorManagerInterface.class), new Configuration(), new SystemClock(),
mock(TaskHeartbeatHandler.class), mock(AppContext.class),
true, Resource.newInstance(1024, 1), createFakeContainerContext(), false);
@@ -243,7 +243,7 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(
TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1);
TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
- mock(TaskAttemptListener.class), new Configuration(),
+ mock(TaskCommunicatorManagerInterface.class), new Configuration(),
new SystemClock(), mock(TaskHeartbeatHandler.class),
mock(AppContext.class), false, Resource.newInstance(1024,
1), createFakeContainerContext(), false);
@@ -285,7 +285,7 @@ public class TestTaskAttempt {
TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 0);
MockEventHandler eventHandler = new MockEventHandler();
- TaskAttemptListener taListener = createMockTaskAttemptListener();
+ TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -334,7 +334,7 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
MockEventHandler eventHandler = spy(new MockEventHandler());
- TaskAttemptListener taListener = createMockTaskAttemptListener();
+ TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -354,7 +354,7 @@ public class TestTaskAttempt {
AppContext appCtx = mock(AppContext.class);
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appCtx);
containers.addContainerIfNew(container, 0, 0, 0);
@@ -434,7 +434,7 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
MockEventHandler eventHandler = new MockEventHandler();
- TaskAttemptListener taListener = createMockTaskAttemptListener();
+ TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -454,7 +454,7 @@ public class TestTaskAttempt {
AppContext appCtx = mock(AppContext.class);
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appCtx);
containers.addContainerIfNew(container, 0, 0, 0);
@@ -498,7 +498,7 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
MockEventHandler eventHandler = spy(new MockEventHandler());
- TaskAttemptListener taListener = createMockTaskAttemptListener();
+ TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -518,7 +518,7 @@ public class TestTaskAttempt {
AppContext appCtx = mock(AppContext.class);
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appCtx);
containers.addContainerIfNew(container, 0, 0, 0);
@@ -589,7 +589,7 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
MockEventHandler eventHandler = spy(new MockEventHandler());
- TaskAttemptListener taListener = createMockTaskAttemptListener();
+ TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -610,7 +610,7 @@ public class TestTaskAttempt {
AppContext appCtx = mock(AppContext.class);
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appCtx);
containers.addContainerIfNew(container, 0, 0, 0);
@@ -719,7 +719,7 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
MockEventHandler eventHandler = spy(new MockEventHandler());
- TaskAttemptListener taListener = createMockTaskAttemptListener();
+ TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -740,7 +740,7 @@ public class TestTaskAttempt {
AppContext appCtx = mock(AppContext.class);
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appCtx);
containers.addContainerIfNew(container, 0, 0, 0);
@@ -810,7 +810,7 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
MockEventHandler eventHandler = spy(new MockEventHandler());
- TaskAttemptListener taListener = createMockTaskAttemptListener();
+ TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -830,7 +830,7 @@ public class TestTaskAttempt {
AppContext appCtx = mock(AppContext.class);
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appCtx);
containers.addContainerIfNew(container, 0, 0, 0);
@@ -904,7 +904,7 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
MockEventHandler eventHandler = spy(new MockEventHandler());
- TaskAttemptListener taListener = createMockTaskAttemptListener();
+ TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -924,7 +924,7 @@ public class TestTaskAttempt {
AppContext appCtx = mock(AppContext.class);
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appCtx);
containers.addContainerIfNew(container, 0, 0, 0);
@@ -1006,7 +1006,7 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
MockEventHandler eventHandler = spy(new MockEventHandler());
- TaskAttemptListener taListener = createMockTaskAttemptListener();
+ TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -1026,7 +1026,7 @@ public class TestTaskAttempt {
AppContext appCtx = mock(AppContext.class);
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appCtx);
containers.addContainerIfNew(container, 0, 0, 0);
@@ -1105,7 +1105,7 @@ public class TestTaskAttempt {
MockEventHandler mockEh = new MockEventHandler();
MockEventHandler eventHandler = spy(mockEh);
- TaskAttemptListener taListener = createMockTaskAttemptListener();
+ TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -1125,7 +1125,7 @@ public class TestTaskAttempt {
AppContext appCtx = mock(AppContext.class);
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appCtx);
containers.addContainerIfNew(container, 0, 0, 0);
@@ -1249,7 +1249,7 @@ public class TestTaskAttempt {
TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
MockEventHandler eventHandler = spy(new MockEventHandler());
- TaskAttemptListener taListener = createMockTaskAttemptListener();
+ TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
@@ -1268,7 +1268,7 @@ public class TestTaskAttempt {
AppContext appCtx = mock(AppContext.class);
AMContainerMap containers = new AMContainerMap(
- mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
new ContainerContextMatcher(), appCtx);
containers.addContainerIfNew(container, 0, 0, 0);
@@ -1324,7 +1324,7 @@ public class TestTaskAttempt {
public int taskAttemptStartedEventLogged = 0;
public int taskAttemptFinishedEventLogged = 0;
public MockTaskAttemptImpl(TezTaskID taskId, int attemptNumber,
- EventHandler eventHandler, TaskAttemptListener tal,
+ EventHandler eventHandler, TaskCommunicatorManagerInterface tal,
Configuration conf, Clock clock,
TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
boolean isRescheduled,
@@ -1378,8 +1378,8 @@ public class TestTaskAttempt {
new Credentials(), new HashMap<String, String>(), "");
}
- private TaskAttemptListener createMockTaskAttemptListener() {
- TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+ private TaskCommunicatorManagerInterface createMockTaskAttemptListener() {
+ TaskCommunicatorManagerInterface taListener = mock(TaskCommunicatorManagerInterface.class);
TaskCommunicator taskComm = mock(TaskCommunicator.class);
doReturn(new InetSocketAddress("localhost", 0)).when(taskComm).getAddress();
doReturn(taskComm).when(taListener).getTaskCommunicator(0);
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
index 4a797e0..6bbfc3d 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttemptRecovery.java
@@ -42,7 +42,7 @@ import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttemptStateInternal;
@@ -146,7 +146,7 @@ public class TestTaskAttemptRecovery {
TezTaskID.fromString("task_1407371892933_0001_1_00_000000");
ta =
new TaskAttemptImpl(taskId, 0, mockEventHandler,
- mock(TaskAttemptListener.class), new Configuration(),
+ mock(TaskCommunicatorManagerInterface.class), new Configuration(),
new SystemClock(), mock(TaskHeartbeatHandler.class),
mockAppContext, false, Resource.newInstance(1, 1),
mock(ContainerContext.class), false, mockTask);
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 807f277..24c9664 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -50,7 +50,7 @@ import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.StateChangeNotifier;
import org.apache.tez.dag.app.dag.TaskStateInternal;
@@ -85,7 +85,7 @@ public class TestTaskImpl {
private final int partition = 1;
private Configuration conf;
- private TaskAttemptListener taskAttemptListener;
+ private TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
private TaskHeartbeatHandler taskHeartbeatHandler;
private Credentials credentials;
private Clock clock;
@@ -122,7 +122,7 @@ public class TestTaskImpl {
@Before
public void setup() {
conf = new Configuration();
- taskAttemptListener = mock(TaskAttemptListener.class);
+ taskCommunicatorManagerInterface = mock(TaskCommunicatorManagerInterface.class);
taskHeartbeatHandler = mock(TaskHeartbeatHandler.class);
credentials = new Credentials();
clock = new SystemClock();
@@ -151,7 +151,7 @@ public class TestTaskImpl {
eventHandler = new TestEventHandler();
mockTask = new MockTaskImpl(vertexId, partition,
- eventHandler, conf, taskAttemptListener, clock,
+ eventHandler, conf, taskCommunicatorManagerInterface, clock,
taskHeartbeatHandler, appContext, leafVertex,
taskResource, containerContext, vertex);
mockTaskSpec = mock(TaskSpec.class);
@@ -698,11 +698,11 @@ public class TestTaskImpl {
public MockTaskImpl(TezVertexID vertexId, int partition,
EventHandler eventHandler, Configuration conf,
- TaskAttemptListener taskAttemptListener, Clock clock,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Clock clock,
TaskHeartbeatHandler thh, AppContext appContext, boolean leafVertex,
Resource resource,
ContainerContext containerContext, Vertex vertex) {
- super(vertexId, partition, eventHandler, conf, taskAttemptListener,
+ super(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface,
clock, thh, appContext, leafVertex, resource,
containerContext, mock(StateChangeNotifier.class), vertex);
this.vertex = vertex;
@@ -711,7 +711,7 @@ public class TestTaskImpl {
@Override
protected TaskAttemptImpl createAttempt(int attemptNumber, TezTaskAttemptID schedCausalTA) {
MockTaskAttemptImpl attempt = new MockTaskAttemptImpl(getTaskId(),
- attemptNumber, eventHandler, taskAttemptListener,
+ attemptNumber, eventHandler, taskCommunicatorManagerInterface,
conf, clock, taskHeartbeatHandler, appContext,
true, taskResource, containerContext, schedCausalTA);
taskAttempts.add(attempt);
@@ -757,7 +757,7 @@ public class TestTaskImpl {
private TaskAttemptState state = TaskAttemptState.NEW;
public MockTaskAttemptImpl(TezTaskID taskId, int attemptNumber,
- EventHandler eventHandler, TaskAttemptListener tal, Configuration conf,
+ EventHandler eventHandler, TaskCommunicatorManagerInterface tal, Configuration conf,
Clock clock, TaskHeartbeatHandler thh, AppContext appContext,
boolean isRescheduled,
Resource resource, ContainerContext containerContext, TezTaskAttemptID schedCausalTA) {
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
index 1d22e06..eca8274 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskRecovery.java
@@ -46,7 +46,7 @@ import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.StateChangeNotifier;
import org.apache.tez.dag.app.dag.TaskAttemptStateInternal;
@@ -189,7 +189,7 @@ public class TestTaskRecovery {
when(mockAppContext.getHistoryHandler()).thenReturn(mockHistoryEventHandler);
task =
new TaskImpl(vertexId, 0, dispatcher.getEventHandler(),
- new Configuration(), mock(TaskAttemptListener.class),
+ new Configuration(), mock(TaskCommunicatorManagerInterface.class),
new SystemClock(), mock(TaskHeartbeatHandler.class),
mockAppContext, false, Resource.newInstance(1, 1),
mock(ContainerContext.class), mock(StateChangeNotifier.class), vertex);
[7/7] tez git commit: TEZ-2708. Rename classes and variables post
TEZ-2003 changes. (sseth)
Posted by ss...@apache.org.
TEZ-2708. Rename classes and variables post TEZ-2003 changes. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/8b278ea8
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/8b278ea8
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/8b278ea8
Branch: refs/heads/master
Commit: 8b278ea84f4c64e7144c07fa79b38a6a719541d2
Parents: dc0ee01
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Aug 25 16:48:00 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Aug 25 16:48:00 2015 -0700
----------------------------------------------------------------------
tez-dag/findbugs-exclude.xml | 15 +-
.../java/org/apache/tez/dag/app/AppContext.java | 4 +-
.../dag/app/ContainerLauncherContextImpl.java | 4 +-
.../org/apache/tez/dag/app/DAGAppMaster.java | 113 +--
.../apache/tez/dag/app/TaskAttemptListener.java | 46 --
.../dag/app/TaskAttemptListenerImpTezDag.java | 449 -----------
.../dag/app/TaskCommunicatorContextImpl.java | 22 +-
.../tez/dag/app/TaskCommunicatorManager.java | 449 +++++++++++
.../app/TaskCommunicatorManagerInterface.java | 46 ++
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 10 +-
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 8 +-
.../apache/tez/dag/app/dag/impl/TaskImpl.java | 10 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 10 +-
.../dag/app/launcher/ContainerLauncherImpl.java | 410 ----------
.../app/launcher/ContainerLauncherManager.java | 217 +++++
.../app/launcher/ContainerLauncherRouter.java | 215 -----
.../app/launcher/LocalContainerLauncher.java | 8 +-
.../app/launcher/TezContainerLauncherImpl.java | 410 ++++++++++
.../tez/dag/app/rm/ContainerLauncherEvent.java | 116 +++
.../dag/app/rm/ContainerLauncherEventType.java | 25 +
.../rm/ContainerLauncherLaunchRequestEvent.java | 79 ++
.../rm/ContainerLauncherStopRequestEvent.java | 34 +
.../tez/dag/app/rm/NMCommunicatorEvent.java | 115 ---
.../tez/dag/app/rm/NMCommunicatorEventType.java | 25 -
.../rm/NMCommunicatorLaunchRequestEvent.java | 78 --
.../app/rm/NMCommunicatorStopRequestEvent.java | 33 -
.../dag/app/rm/TaskSchedulerContextImpl.java | 29 +-
.../dag/app/rm/TaskSchedulerEventHandler.java | 784 ------------------
.../tez/dag/app/rm/TaskSchedulerManager.java | 786 +++++++++++++++++++
.../dag/app/rm/container/AMContainerImpl.java | 26 +-
.../dag/app/rm/container/AMContainerMap.java | 6 +-
.../apache/tez/dag/app/MockDAGAppMaster.java | 23 +-
.../tez/dag/app/TestMockDAGAppMaster.java | 2 +-
.../app/TestTaskAttemptListenerImplTezDag.java | 426 ----------
.../app/TestTaskAttemptListenerImplTezDag2.java | 137 ----
.../app/TestTaskCommunicatorContextImpl.java | 2 +-
.../dag/app/TestTaskCommunicatorManager.java | 2 +-
.../dag/app/TestTaskCommunicatorManager1.java | 425 ++++++++++
.../dag/app/TestTaskCommunicatorManager2.java | 136 ++++
.../apache/tez/dag/app/dag/impl/TestCommit.java | 6 +-
.../tez/dag/app/dag/impl/TestDAGImpl.java | 12 +-
.../tez/dag/app/dag/impl/TestDAGRecovery.java | 4 +-
.../tez/dag/app/dag/impl/TestTaskAttempt.java | 58 +-
.../app/dag/impl/TestTaskAttemptRecovery.java | 4 +-
.../tez/dag/app/dag/impl/TestTaskImpl.java | 16 +-
.../tez/dag/app/dag/impl/TestTaskRecovery.java | 4 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 31 +-
.../tez/dag/app/dag/impl/TestVertexImpl2.java | 4 +-
.../dag/app/dag/impl/TestVertexRecovery.java | 10 +-
.../launcher/TestContainerLauncherManager.java | 359 +++++++++
.../launcher/TestContainerLauncherRouter.java | 359 ---------
.../tez/dag/app/rm/TestContainerReuse.java | 310 ++++----
.../app/rm/TestTaskSchedulerEventHandler.java | 707 -----------------
.../dag/app/rm/TestTaskSchedulerHelpers.java | 14 +-
.../dag/app/rm/TestTaskSchedulerManager.java | 708 +++++++++++++++++
.../dag/app/rm/container/TestAMContainer.java | 24 +-
.../app/rm/container/TestAMContainerMap.java | 6 +-
.../tez/dag/app/rm/node/TestAMNodeTracker.java | 38 +-
.../tez/service/impl/ContainerRunnerImpl.java | 23 +-
.../apache/tez/runtime/task/TezTaskRunner.java | 451 -----------
.../tez/runtime/task/TestTaskExecution.java | 362 ---------
.../tez/runtime/task/TestTaskExecution2.java | 2 +-
62 files changed, 4220 insertions(+), 5027 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/tez-dag/findbugs-exclude.xml b/tez-dag/findbugs-exclude.xml
index 9d15035..6db3b7c 100644
--- a/tez-dag/findbugs-exclude.xml
+++ b/tez-dag/findbugs-exclude.xml
@@ -87,13 +87,13 @@
</Match>
<Match>
- <Class name="~org\.apache\.tez\.dag\.app\.rm\.TaskSchedulerEventHandler"/>
+ <Class name="~org\.apache\.tez\.dag\.app\.rm\.TaskSchedulerManager"/>
<Bug pattern="BC_UNCONFIRMED_CAST"/>
</Match>
<Match>
- <Class name="org.apache.tez.dag.app.launcher.ContainerLauncherRouter" />
- <Method name="handle" params="org.apache.tez.dag.app.rm.NMCommunicatorEvent" returns="void" />
+ <Class name="org.apache.tez.dag.app.launcher.ContainerLauncherManager" />
+ <Method name="handle" params="org.apache.tez.dag.app.rm.ContainerLauncherEvent" returns="void" />
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
@@ -151,7 +151,7 @@
<Field name="context"/>
<Field name="currentDAG"/>
<Field name="state"/>
- <Field name="taskSchedulerEventHandler"/>
+ <Field name="taskSchedulerManager"/>
<Field name="versionMismatch"/>
<Field name="versionMismatchDiagnostics"/>
<Field name="containers"/>
@@ -165,13 +165,6 @@
<Bug pattern="IS2_INCONSISTENT_SYNC"/>
</Match>
- <!-- TEZ-1955 -->
- <Match>
- <Class name="org.apache.tez.dag.app.rm.TaskSchedulerEventHandler"/>
- <Field name="taskScheduler"/>
- <Bug pattern="IS2_INCONSISTENT_SYNC"/>
- </Match>
-
<!-- TEZ-1956 -->
<Match>
<Class name="org.apache.tez.dag.app.rm.YarnTaskSchedulerService"/>
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
index 516fcef..657267b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.app.rm.TaskSchedulerEventHandler;
+import org.apache.tez.dag.app.rm.TaskSchedulerManager;
import org.apache.tez.dag.app.rm.container.AMContainerMap;
import org.apache.tez.dag.app.rm.node.AMNodeTracker;
import org.apache.tez.common.security.ACLManager;
@@ -88,7 +88,7 @@ public interface AppContext {
AMNodeTracker getNodeTracker();
- TaskSchedulerEventHandler getTaskScheduler();
+ TaskSchedulerManager getTaskScheduler();
boolean isSession();
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java
index 3a2efc5..20bfd13 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java
@@ -33,10 +33,10 @@ import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
public class ContainerLauncherContextImpl implements ContainerLauncherContext {
private final AppContext context;
- private final TaskAttemptListener tal;
+ private final TaskCommunicatorManagerInterface tal;
private final UserPayload initialUserPayload;
- public ContainerLauncherContextImpl(AppContext appContext, TaskAttemptListener tal, UserPayload initialUserPayload) {
+ public ContainerLauncherContextImpl(AppContext appContext, TaskCommunicatorManagerInterface tal, UserPayload initialUserPayload) {
this.context = appContext;
this.tal = tal;
this.initialUserPayload = initialUserPayload;
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 84b3095..34e7c2a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -40,7 +40,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -149,10 +148,10 @@ import org.apache.tez.dag.app.dag.event.TaskEventType;
import org.apache.tez.dag.app.dag.event.VertexEvent;
import org.apache.tez.dag.app.dag.event.VertexEventType;
import org.apache.tez.dag.app.dag.impl.DAGImpl;
-import org.apache.tez.dag.app.launcher.ContainerLauncherRouter;
+import org.apache.tez.dag.app.launcher.ContainerLauncherManager;
import org.apache.tez.dag.app.rm.AMSchedulerEventType;
-import org.apache.tez.dag.app.rm.NMCommunicatorEventType;
-import org.apache.tez.dag.app.rm.TaskSchedulerEventHandler;
+import org.apache.tez.dag.app.rm.ContainerLauncherEventType;
+import org.apache.tez.dag.app.rm.TaskSchedulerManager;
import org.apache.tez.dag.app.rm.container.AMContainerEventType;
import org.apache.tez.dag.app.rm.container.AMContainerMap;
import org.apache.tez.dag.app.rm.container.ContainerContextMatcher;
@@ -236,16 +235,16 @@ public class DAGAppMaster extends AbstractService {
private AppContext context;
private Configuration amConf;
private AsyncDispatcher dispatcher;
- private ContainerLauncherRouter containerLauncherRouter;
+ private ContainerLauncherManager containerLauncherManager;
private ContainerHeartbeatHandler containerHeartbeatHandler;
private TaskHeartbeatHandler taskHeartbeatHandler;
- private TaskAttemptListener taskAttemptListener;
+ private TaskCommunicatorManagerInterface taskCommunicatorManager;
private JobTokenSecretManager jobTokenSecretManager =
new JobTokenSecretManager();
private Token<JobTokenIdentifier> sessionToken;
private DagEventDispatcher dagEventDispatcher;
private VertexEventDispatcher vertexEventDispatcher;
- private TaskSchedulerEventHandler taskSchedulerEventHandler;
+ private TaskSchedulerManager taskSchedulerManager;
private WebUIService webUIService;
private HistoryEventHandler historyEventHandler;
private final Map<String, LocalResource> amResources = new HashMap<String, LocalResource>();
@@ -472,13 +471,13 @@ public class DAGAppMaster extends AbstractService {
//service to handle requests to TaskUmbilicalProtocol
- taskAttemptListener = createTaskAttemptListener(context,
+ taskCommunicatorManager = createTaskCommunicatorManager(context,
taskHeartbeatHandler, containerHeartbeatHandler, taskCommunicatorDescriptors);
- addIfService(taskAttemptListener, true);
+ addIfService(taskCommunicatorManager, true);
containerSignatureMatcher = createContainerSignatureMatcher();
containers = new AMContainerMap(containerHeartbeatHandler,
- taskAttemptListener, containerSignatureMatcher, context);
+ taskCommunicatorManager, containerSignatureMatcher, context);
addIfService(containers, true);
dispatcher.register(AMContainerEventType.class, containers);
@@ -521,29 +520,30 @@ public class DAGAppMaster extends AbstractService {
- this.taskSchedulerEventHandler = new TaskSchedulerEventHandler(context,
+ this.taskSchedulerManager = new TaskSchedulerManager(context,
clientRpcServer, dispatcher.getEventHandler(), containerSignatureMatcher, webUIService,
taskSchedulerDescriptors, isLocal);
- addIfService(taskSchedulerEventHandler, true);
+ addIfService(taskSchedulerManager, true);
if (enableWebUIService()) {
- addIfServiceDependency(taskSchedulerEventHandler, webUIService);
+ addIfServiceDependency(taskSchedulerManager, webUIService);
}
if (isLastAMRetry) {
LOG.info("AM will unregister as this is the last attempt"
+ ", currentAttempt=" + appAttemptID.getAttemptId()
+ ", maxAttempts=" + maxAppAttempts);
- this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+ this.taskSchedulerManager.setShouldUnregisterFlag();
}
dispatcher.register(AMSchedulerEventType.class,
- taskSchedulerEventHandler);
- addIfServiceDependency(taskSchedulerEventHandler, clientRpcServer);
+ taskSchedulerManager);
+ addIfServiceDependency(taskSchedulerManager, clientRpcServer);
- this.containerLauncherRouter = createContainerLauncherRouter(containerLauncherDescriptors, isLocal);
- addIfService(containerLauncherRouter, true);
- dispatcher.register(NMCommunicatorEventType.class, containerLauncherRouter);
+ this.containerLauncherManager = createContainerLauncherManager(containerLauncherDescriptors,
+ isLocal);
+ addIfService(containerLauncherManager, true);
+ dispatcher.register(ContainerLauncherEventType.class, containerLauncherManager);
historyEventHandler = createHistoryEventHandler(context);
addIfService(historyEventHandler, true);
@@ -632,8 +632,8 @@ public class DAGAppMaster extends AbstractService {
}
@VisibleForTesting
- protected TaskSchedulerEventHandler getTaskSchedulerEventHandler() {
- return taskSchedulerEventHandler;
+ protected TaskSchedulerManager getTaskSchedulerManager() {
+ return taskSchedulerManager;
}
@VisibleForTesting
@@ -661,7 +661,7 @@ public class DAGAppMaster extends AbstractService {
sendEvent(new DAGEvent(currentDAG.getID(), DAGEventType.INTERNAL_ERROR));
} else {
LOG.info("Internal Error. Finishing directly as no dag is active.");
- this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+ this.taskSchedulerManager.setShouldUnregisterFlag();
shutdownHandler.shutdown();
}
break;
@@ -673,7 +673,7 @@ public class DAGAppMaster extends AbstractService {
System.out.println(timeStamp + " Completed Dag: " + finishEvt.getDAGId().toString());
if (!isSession) {
LOG.info("Not a session, AM will unregister as DAG has completed");
- this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+ this.taskSchedulerManager.setShouldUnregisterFlag();
_updateLoggers(currentDAG, "_post");
setStateOnDAGCompletion();
LOG.info("Shutting down on completion of dag:" +
@@ -722,7 +722,7 @@ public class DAGAppMaster extends AbstractService {
+ finishEvt.getDAGState()
+ ". Error. Shutting down.");
state = DAGAppMasterState.ERROR;
- this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+ this.taskSchedulerManager.setShouldUnregisterFlag();
shutdownHandler.shutdown();
break;
}
@@ -741,11 +741,11 @@ public class DAGAppMaster extends AbstractService {
// Leaving the taskSchedulerEventHandler here for now. Doesn't generate new events.
// However, eventually it needs to be moved out.
- this.taskSchedulerEventHandler.dagCompleted();
+ this.taskSchedulerManager.dagCompleted();
state = DAGAppMasterState.IDLE;
} else {
LOG.info("Session shutting down now.");
- this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+ this.taskSchedulerManager.setShouldUnregisterFlag();
if (this.historyEventHandler.hasRecoveryFailed()) {
state = DAGAppMasterState.FAILED;
} else {
@@ -772,8 +772,8 @@ public class DAGAppMaster extends AbstractService {
DAGAppMasterEventDagCleanup cleanupEvent = (DAGAppMasterEventDagCleanup) event;
LOG.info("Cleaning up DAG: name=" + cleanupEvent.getDag().getName() + ", with id=" +
cleanupEvent.getDag().getID());
- containerLauncherRouter.dagComplete(cleanupEvent.getDag());
- taskAttemptListener.dagComplete(cleanupEvent.getDag());
+ containerLauncherManager.dagComplete(cleanupEvent.getDag());
+ taskCommunicatorManager.dagComplete(cleanupEvent.getDag());
nodes.dagComplete(cleanupEvent.getDag());
containers.dagComplete(cleanupEvent.getDag());
TezTaskAttemptID.clearCache();
@@ -785,9 +785,9 @@ public class DAGAppMaster extends AbstractService {
break;
case NEW_DAG_SUBMITTED:
// Inform sub-components that a new DAG has been submitted.
- taskSchedulerEventHandler.dagSubmitted();
- containerLauncherRouter.dagSubmitted();
- taskAttemptListener.dagSubmitted();
+ taskSchedulerManager.dagSubmitted();
+ containerLauncherManager.dagSubmitted();
+ taskCommunicatorManager.dagSubmitted();
break;
default:
throw new TezUncheckedException(
@@ -917,7 +917,7 @@ public class DAGAppMaster extends AbstractService {
// create single dag
DAGImpl newDag =
new DAGImpl(dagId, amConf, dagPB, dispatcher.getEventHandler(),
- taskAttemptListener, dagCredentials, clock,
+ taskCommunicatorManager, dagCredentials, clock,
appMasterUgi.getShortUserName(),
taskHeartbeatHandler, context);
@@ -1048,13 +1048,13 @@ public class DAGAppMaster extends AbstractService {
}
}
- protected TaskAttemptListener createTaskAttemptListener(AppContext context,
- TaskHeartbeatHandler thh,
- ContainerHeartbeatHandler chh,
- List<NamedEntityDescriptor> entityDescriptors) {
- TaskAttemptListener lis =
- new TaskAttemptListenerImpTezDag(context, thh, chh, entityDescriptors);
- return lis;
+ protected TaskCommunicatorManagerInterface createTaskCommunicatorManager(AppContext context,
+ TaskHeartbeatHandler thh,
+ ContainerHeartbeatHandler chh,
+ List<NamedEntityDescriptor> entityDescriptors) {
+ TaskCommunicatorManagerInterface tcm =
+ new TaskCommunicatorManager(context, thh, chh, entityDescriptors);
+ return tcm;
}
protected TaskHeartbeatHandler createTaskHeartbeatHandler(AppContext context,
@@ -1074,10 +1074,11 @@ public class DAGAppMaster extends AbstractService {
return chh;
}
- protected ContainerLauncherRouter createContainerLauncherRouter(List<NamedEntityDescriptor> containerLauncherDescriptors,
- boolean isLocal) throws
+ protected ContainerLauncherManager createContainerLauncherManager(
+ List<NamedEntityDescriptor> containerLauncherDescriptors,
+ boolean isLocal) throws
UnknownHostException {
- return new ContainerLauncherRouter(context, taskAttemptListener, workingDirectory,
+ return new ContainerLauncherManager(context, taskCommunicatorManager, workingDirectory,
containerLauncherDescriptors, isLocal);
}
@@ -1101,12 +1102,12 @@ public class DAGAppMaster extends AbstractService {
return dispatcher;
}
- public ContainerLauncherRouter getContainerLauncherRouter() {
- return containerLauncherRouter;
+ public ContainerLauncherManager getContainerLauncherManager() {
+ return containerLauncherManager;
}
- public TaskAttemptListener getTaskAttemptListener() {
- return taskAttemptListener;
+ public TaskCommunicatorManagerInterface getTaskCommunicatorManager() {
+ return taskCommunicatorManager;
}
public ContainerId getAppContainerId() {
@@ -1213,7 +1214,7 @@ public class DAGAppMaster extends AbstractService {
public void shutdownTezAM() throws TezException {
sessionStopped.set(true);
synchronized (this) {
- this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+ this.taskSchedulerManager.setShouldUnregisterFlag();
if (currentDAG != null
&& !currentDAG.isComplete()) {
//send a DAG_KILL message
@@ -1473,8 +1474,8 @@ public class DAGAppMaster extends AbstractService {
}
@Override
- public TaskSchedulerEventHandler getTaskScheduler() {
- return taskSchedulerEventHandler;
+ public TaskSchedulerManager getTaskScheduler() {
+ return taskSchedulerManager;
}
@Override
@@ -1574,7 +1575,7 @@ public class DAGAppMaster extends AbstractService {
throw new TezUncheckedException(
"Cannot get ApplicationACLs before all services have started");
}
- return taskSchedulerEventHandler.getApplicationAcls();
+ return taskSchedulerManager.getApplicationAcls();
}
@Override
@@ -1805,7 +1806,7 @@ public class DAGAppMaster extends AbstractService {
if (versionMismatch) {
// Short-circuit and return as no DAG should not be run
- this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+ this.taskSchedulerManager.setShouldUnregisterFlag();
shutdownHandler.shutdown();
return;
}
@@ -1825,7 +1826,7 @@ public class DAGAppMaster extends AbstractService {
LOG.error("Error occurred when trying to recover data from previous attempt."
+ " Shutting down AM", e);
this.state = DAGAppMasterState.ERROR;
- this.taskSchedulerEventHandler.setShouldUnregisterFlag();
+ this.taskSchedulerManager.setShouldUnregisterFlag();
shutdownHandler.shutdown();
return;
}
@@ -1929,7 +1930,7 @@ public class DAGAppMaster extends AbstractService {
private void initiateStop() {
- taskSchedulerEventHandler.initiateStop();
+ taskSchedulerManager.initiateStop();
}
@Override
@@ -1954,8 +1955,8 @@ public class DAGAppMaster extends AbstractService {
LOG.debug("Checking whether tez scratch data dir should be deleted, deleteTezScratchData="
+ deleteTezScratchData);
}
- if (deleteTezScratchData && this.taskSchedulerEventHandler != null
- && this.taskSchedulerEventHandler.hasUnregistered()) {
+ if (deleteTezScratchData && this.taskSchedulerManager != null
+ && this.taskSchedulerManager.hasUnregistered()) {
// Delete tez scratch data dir
if (this.tezSystemStagingDir != null) {
try {
@@ -2208,7 +2209,7 @@ public class DAGAppMaster extends AbstractService {
// Notify TaskScheduler that a SIGTERM has been received so that it
// unregisters quickly with proper status
LOG.info("DAGAppMaster received a signal. Signaling TaskScheduler");
- appMaster.taskSchedulerEventHandler.setSignalled(true);
+ appMaster.taskSchedulerManager.setSignalled(true);
}
if (EnumSet.of(DAGAppMasterState.NEW, DAGAppMasterState.INITED,
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
deleted file mode 100644
index 761bdb0..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.serviceplugins.api.ContainerEndReason;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.api.TaskCommunicator;
-import org.apache.tez.dag.app.rm.container.AMContainerTask;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-/**
- * This class listens for changes to the state of a Task.
- */
-public interface TaskAttemptListener {
-
- void registerRunningContainer(ContainerId containerId, int taskCommId);
-
- void registerTaskAttempt(AMContainerTask amContainerTask, ContainerId containerId, int taskCommId);
-
- void unregisterRunningContainer(ContainerId containerId, int taskCommId, ContainerEndReason endReason, String diagnostics);
-
- void unregisterTaskAttempt(TezTaskAttemptID attemptID, int taskCommId, TaskAttemptEndReason endReason, String diagnostics);
-
- void dagComplete(DAG dag);
-
- void dagSubmitted();
-
- TaskCommunicator getTaskCommunicator(int taskCommIndex);
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
deleted file mode 100644
index 185193f..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ /dev/null
@@ -1,449 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.tez.dag.app;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.commons.collections4.ListUtils;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
-import org.apache.tez.serviceplugins.api.ContainerEndReason;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEvent;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate;
-import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent;
-import org.apache.tez.runtime.api.impl.EventType;
-import org.apache.tez.dag.api.event.VertexStateUpdate;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.tez.common.ReflectionUtils;
-import org.apache.tez.common.TezUtilsInternal;
-import org.apache.tez.dag.api.TaskCommunicator;
-import org.apache.tez.dag.api.TaskCommunicatorContext;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.dag.api.TaskHeartbeatResponse;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.TaskHeartbeatRequest;
-import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.app.dag.Task;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptKilled;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
-import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
-import org.apache.tez.dag.app.rm.container.AMContainerTask;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezVertexID;
-import org.apache.tez.runtime.api.impl.TezEvent;
-
-
-@SuppressWarnings("unchecked")
-@InterfaceAudience.Private
-public class TaskAttemptListenerImpTezDag extends AbstractService implements
- TaskAttemptListener {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(TaskAttemptListenerImpTezDag.class);
-
- private final AppContext context;
- private final TaskCommunicator[] taskCommunicators;
- private final TaskCommunicatorContext[] taskCommunicatorContexts;
- protected final ServicePluginLifecycleAbstractService []taskCommunicatorServiceWrappers;
-
- protected final TaskHeartbeatHandler taskHeartbeatHandler;
- protected final ContainerHeartbeatHandler containerHeartbeatHandler;
-
- private final TaskHeartbeatResponse RESPONSE_SHOULD_DIE = new TaskHeartbeatResponse(true, null, 0, 0);
-
- private final ConcurrentMap<TezTaskAttemptID, ContainerId> registeredAttempts =
- new ConcurrentHashMap<TezTaskAttemptID, ContainerId>();
- private final ConcurrentMap<ContainerId, ContainerInfo> registeredContainers =
- new ConcurrentHashMap<ContainerId, ContainerInfo>();
-
- // Defined primarily to work around ConcurrentMaps not accepting null values
- private static final class ContainerInfo {
- TezTaskAttemptID taskAttemptId;
- ContainerInfo(TezTaskAttemptID taskAttemptId) {
- this.taskAttemptId = taskAttemptId;
- }
- }
-
- private static final ContainerInfo NULL_CONTAINER_INFO = new ContainerInfo(null);
-
-
- public TaskAttemptListenerImpTezDag(AppContext context,
- TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
- List<NamedEntityDescriptor> taskCommunicatorDescriptors) {
- super(TaskAttemptListenerImpTezDag.class.getName());
- this.context = context;
- this.taskHeartbeatHandler = thh;
- this.containerHeartbeatHandler = chh;
- Preconditions.checkArgument(
- taskCommunicatorDescriptors != null && !taskCommunicatorDescriptors.isEmpty(),
- "TaskCommunicators must be specified");
- this.taskCommunicators = new TaskCommunicator[taskCommunicatorDescriptors.size()];
- this.taskCommunicatorContexts = new TaskCommunicatorContext[taskCommunicatorDescriptors.size()];
- this.taskCommunicatorServiceWrappers = new ServicePluginLifecycleAbstractService[taskCommunicatorDescriptors.size()];
- for (int i = 0 ; i < taskCommunicatorDescriptors.size() ; i++) {
- UserPayload userPayload = taskCommunicatorDescriptors.get(i).getUserPayload();
- taskCommunicatorContexts[i] = new TaskCommunicatorContextImpl(context, this, userPayload, i);
- taskCommunicators[i] = createTaskCommunicator(taskCommunicatorDescriptors.get(i), i);
- taskCommunicatorServiceWrappers[i] = new ServicePluginLifecycleAbstractService(taskCommunicators[i]);
- }
- // TODO TEZ-2118 Start using taskCommunicator indices properly
- }
-
- @Override
- public void serviceStart() {
- // TODO Why is init tied to serviceStart
- for (int i = 0 ; i < taskCommunicators.length ; i++) {
- taskCommunicatorServiceWrappers[i].init(getConfig());
- taskCommunicatorServiceWrappers[i].start();
- }
- }
-
- @Override
- public void serviceStop() {
- for (int i = 0 ; i < taskCommunicators.length ; i++) {
- taskCommunicatorServiceWrappers[i].stop();
- }
- }
-
- @VisibleForTesting
- TaskCommunicator createTaskCommunicator(NamedEntityDescriptor taskCommDescriptor,
- int taskCommIndex) {
- if (taskCommDescriptor.getEntityName().equals(TezConstants.getTezYarnServicePluginName())) {
- return createDefaultTaskCommunicator(taskCommunicatorContexts[taskCommIndex]);
- } else if (taskCommDescriptor.getEntityName()
- .equals(TezConstants.getTezUberServicePluginName())) {
- return createUberTaskCommunicator(taskCommunicatorContexts[taskCommIndex]);
- } else {
- return createCustomTaskCommunicator(taskCommunicatorContexts[taskCommIndex],
- taskCommDescriptor);
- }
- }
-
- @VisibleForTesting
- TaskCommunicator createDefaultTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
- LOG.info("Using Default Task Communicator");
- return new TezTaskCommunicatorImpl(taskCommunicatorContext);
- }
-
- @VisibleForTesting
- TaskCommunicator createUberTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
- LOG.info("Using Default Local Task Communicator");
- return new TezLocalTaskCommunicatorImpl(taskCommunicatorContext);
- }
-
- @VisibleForTesting
- TaskCommunicator createCustomTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext,
- NamedEntityDescriptor taskCommDescriptor) {
- LOG.info("Using TaskCommunicator {}:{} " + taskCommDescriptor.getEntityName(),
- taskCommDescriptor.getClassName());
- Class<? extends TaskCommunicator> taskCommClazz =
- (Class<? extends TaskCommunicator>) ReflectionUtils
- .getClazz(taskCommDescriptor.getClassName());
- try {
- Constructor<? extends TaskCommunicator> ctor =
- taskCommClazz.getConstructor(TaskCommunicatorContext.class);
- return ctor.newInstance(taskCommunicatorContext);
- } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
- throw new TezUncheckedException(e);
- }
- }
-
- public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request)
- throws IOException, TezException {
- ContainerId containerId = ConverterUtils.toContainerId(request
- .getContainerIdentifier());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Received heartbeat from container"
- + ", request=" + request);
- }
-
- if (!registeredContainers.containsKey(containerId)) {
- LOG.warn("Received task heartbeat from unknown container with id: " + containerId +
- ", asking it to die");
- return RESPONSE_SHOULD_DIE;
- }
-
- // A heartbeat can come in anytime. The AM may have made a decision to kill a running task/container
- // meanwhile. If the decision is processed through the pipeline before the heartbeat is processed,
- // the heartbeat will be dropped. Otherwise the heartbeat will be processed - and the system
- // know how to handle this - via FailedInputEvents for example (relevant only if the heartbeat has events).
- // So - avoiding synchronization.
-
- pingContainerHeartbeatHandler(containerId);
- TaskAttemptEventInfo eventInfo = new TaskAttemptEventInfo(0, null, 0);
- TezTaskAttemptID taskAttemptID = request.getTaskAttemptId();
- if (taskAttemptID != null) {
- ContainerId containerIdFromMap = registeredAttempts.get(taskAttemptID);
- if (containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
- // This can happen when a task heartbeats. Meanwhile the container is unregistered.
- // The information will eventually make it through to the plugin via a corresponding unregister.
- // There's a race in that case between the unregister making it through, and this method returning.
- // TODO TEZ-2003 (post) TEZ-2666. An exception back is likely a better approach than sending a shouldDie = true,
- // so that the plugin can handle the scenario. Alternately augment the response with error codes.
- // Error codes would be better than exceptions.
- LOG.info("Attempt: " + taskAttemptID + " is not recognized for heartbeats");
- return RESPONSE_SHOULD_DIE;
- }
-
- List<TezEvent> inEvents = request.getEvents();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Ping from " + taskAttemptID.toString() +
- " events: " + (inEvents != null ? inEvents.size() : -1));
- }
-
- long currTime = context.getClock().getTime();
- List<TezEvent> otherEvents = new ArrayList<TezEvent>();
- // route TASK_STATUS_UPDATE_EVENT directly to TaskAttempt and route other events
- // (DATA_MOVEMENT_EVENT, TASK_ATTEMPT_COMPLETED_EVENT, TASK_ATTEMPT_FAILED_EVENT)
- // to VertexImpl to ensure the events ordering
- // 1. DataMovementEvent is logged as RecoveryEvent before TaskAttemptFinishedEvent
- // 2. TaskStatusEvent is handled before TaskAttemptFinishedEvent
- for (TezEvent tezEvent : ListUtils.emptyIfNull(inEvents)) {
- // for now, set the event time on the AM when it is received.
- // this avoids any time disparity between machines.
- tezEvent.setEventReceivedTime(currTime);
- final EventType eventType = tezEvent.getEventType();
- if (eventType == EventType.TASK_STATUS_UPDATE_EVENT) {
- TaskAttemptEvent taskAttemptEvent = new TaskAttemptEventStatusUpdate(taskAttemptID,
- (TaskStatusUpdateEvent) tezEvent.getEvent());
- context.getEventHandler().handle(taskAttemptEvent);
- } else {
- otherEvents.add(tezEvent);
- }
- }
- if(!otherEvents.isEmpty()) {
- TezVertexID vertexId = taskAttemptID.getTaskID().getVertexID();
- context.getEventHandler().handle(
- new VertexEventRouteEvent(vertexId, Collections.unmodifiableList(otherEvents)));
- }
- taskHeartbeatHandler.pinged(taskAttemptID);
- eventInfo = context
- .getCurrentDAG()
- .getVertex(taskAttemptID.getTaskID().getVertexID())
- .getTaskAttemptTezEvents(taskAttemptID, request.getStartIndex(), request.getPreRoutedStartIndex(),
- request.getMaxEvents());
- }
- return new TaskHeartbeatResponse(false, eventInfo.getEvents(), eventInfo.getNextFromEventId(), eventInfo.getNextPreRoutedFromEventId());
- }
- public void taskAlive(TezTaskAttemptID taskAttemptId) {
- taskHeartbeatHandler.pinged(taskAttemptId);
- }
-
- public void containerAlive(ContainerId containerId) {
- pingContainerHeartbeatHandler(containerId);
- }
-
- public void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId) {
- context.getEventHandler()
- .handle(new TaskAttemptEventStartedRemotely(taskAttemptID, containerId, null));
- pingContainerHeartbeatHandler(containerId);
- }
-
- public void taskKilled(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
- String diagnostics) {
- // Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
- // and messages from the scheduler will release the container.
- // TODO TEZ-2003 (post) TEZ-2671 Maybe consider un-registering here itself, since the task is not active anymore,
- // instead of waiting for the unregister to flow through the Container.
- // Fix along the same lines as TEZ-2124 by introducing an explict context.
- context.getEventHandler().handle(new TaskAttemptEventAttemptKilled(taskAttemptId,
- diagnostics, TezUtilsInternal.fromTaskAttemptEndReason(
- taskAttemptEndReason)));
- }
-
- public void taskFailed(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
- String diagnostics) {
- // Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
- // and messages from the scheduler will release the container.
- // TODO TEZ-2003 (post) TEZ-2671 Maybe consider un-registering here itself, since the task is not active anymore,
- // instead of waiting for the unregister to flow through the Container.
- // Fix along the same lines as TEZ-2124 by introducing an explict context.
- context.getEventHandler().handle(new TaskAttemptEventAttemptFailed(taskAttemptId,
- TaskAttemptEventType.TA_FAILED, diagnostics, TezUtilsInternal.fromTaskAttemptEndReason(
- taskAttemptEndReason)));
- }
-
- public void vertexStateUpdateNotificationReceived(VertexStateUpdate event, int taskCommIndex) throws
- Exception {
- taskCommunicators[taskCommIndex].onVertexStateUpdated(event);
- }
-
-
- /**
- * Child checking whether it can commit.
- * <p/>
- * <br/>
- * Repeatedly polls the ApplicationMaster whether it
- * {@link Task#canCommit(TezTaskAttemptID)} This is * a legacy from the
- * centralized commit protocol handling by the JobTracker.
- */
-// @Override
- public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
- LOG.info("Commit go/no-go request from " + taskAttemptId.toString());
- // An attempt is asking if it can commit its output. This can be decided
- // only by the task which is managing the multiple attempts. So redirect the
- // request there.
- taskHeartbeatHandler.progressing(taskAttemptId);
- pingContainerHeartbeatHandler(taskAttemptId);
-
- DAG job = context.getCurrentDAG();
- Task task =
- job.getVertex(taskAttemptId.getTaskID().getVertexID()).
- getTask(taskAttemptId.getTaskID());
- return task.canCommit(taskAttemptId);
- }
-
- // The TaskAttemptListener register / unregister methods in this class are not thread safe.
- // The Tez framework should not invoke these methods from multiple threads.
- @Override
- public void dagComplete(DAG dag) {
- // TODO TEZ-2335. Cleanup TaskHeartbeat handler structures.
- // TODO TEZ-2345. Also cleanup attemptInfo map, so that any tasks which heartbeat are told to die.
- // Container structures remain unchanged - since they could be re-used across restarts.
- // This becomes more relevant when task kills without container kills are allowed.
-
- // TODO TEZ-2336. Send a signal to containers indicating DAG completion.
-
- // Inform all communicators of the dagCompletion.
- for (int i = 0 ; i < taskCommunicators.length ; i++) {
- ((TaskCommunicatorContextImpl)taskCommunicatorContexts[i]).dagCompleteStart(dag);
- taskCommunicators[i].dagComplete(dag.getName());
- ((TaskCommunicatorContextImpl)taskCommunicatorContexts[i]).dagCompleteEnd();
- }
-
- }
-
- @Override
- public void dagSubmitted() {
- // Nothing to do right now. Indicates that a new DAG has been submitted and
- // the context has updated information.
- }
-
- @Override
- public void registerRunningContainer(ContainerId containerId, int taskCommId) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("ContainerId: " + containerId + " registered with TaskAttemptListener");
- }
- ContainerInfo oldInfo = registeredContainers.put(containerId, NULL_CONTAINER_INFO);
- if (oldInfo != null) {
- throw new TezUncheckedException(
- "Multiple registrations for containerId: " + containerId);
- }
- NodeId nodeId = context.getAllContainers().get(containerId).getContainer().getNodeId();
- taskCommunicators[taskCommId].registerRunningContainer(containerId, nodeId.getHost(),
- nodeId.getPort());
- }
-
- @Override
- public void unregisterRunningContainer(ContainerId containerId, int taskCommId, ContainerEndReason endReason, String diagnostics) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Unregistering Container from TaskAttemptListener: " + containerId);
- }
- ContainerInfo containerInfo = registeredContainers.remove(containerId);
- if (containerInfo.taskAttemptId != null) {
- registeredAttempts.remove(containerInfo.taskAttemptId);
- }
- taskCommunicators[taskCommId].registerContainerEnd(containerId, endReason, diagnostics);
- }
-
- @Override
- public void registerTaskAttempt(AMContainerTask amContainerTask,
- ContainerId containerId, int taskCommId) {
- ContainerInfo containerInfo = registeredContainers.get(containerId);
- if (containerInfo == null) {
- throw new TezUncheckedException("Registering task attempt: "
- + amContainerTask.getTask().getTaskAttemptID() + " to unknown container: " + containerId);
- }
- if (containerInfo.taskAttemptId != null) {
- throw new TezUncheckedException("Registering task attempt: "
- + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
- + " with existing assignment to: " +
- containerInfo.taskAttemptId);
- }
-
- // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
- registeredContainers.put(containerId, new ContainerInfo(amContainerTask.getTask().getTaskAttemptID()));
-
- ContainerId containerIdFromMap = registeredAttempts.put(
- amContainerTask.getTask().getTaskAttemptID(), containerId);
- if (containerIdFromMap != null) {
- throw new TezUncheckedException("Registering task attempt: "
- + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
- + " when already assigned to: " + containerIdFromMap);
- }
- taskCommunicators[taskCommId].registerRunningTaskAttempt(containerId, amContainerTask.getTask(),
- amContainerTask.getAdditionalResources(), amContainerTask.getCredentials(),
- amContainerTask.haveCredentialsChanged(), amContainerTask.getPriority());
- }
-
- @Override
- public void unregisterTaskAttempt(TezTaskAttemptID attemptId, int taskCommId, TaskAttemptEndReason endReason, String diagnostics) {
- ContainerId containerId = registeredAttempts.remove(attemptId);
- if (containerId == null) {
- LOG.warn("Unregister task attempt: " + attemptId + " from unknown container");
- return;
- }
- ContainerInfo containerInfo = registeredContainers.get(containerId);
- if (containerInfo == null) {
- LOG.warn("Unregister task attempt: " + attemptId +
- " from non-registered container: " + containerId);
- return;
- }
- // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
- registeredContainers.put(containerId, NULL_CONTAINER_INFO);
- taskCommunicators[taskCommId].unregisterRunningTaskAttempt(attemptId, endReason, diagnostics);
- }
-
- @Override
- public TaskCommunicator getTaskCommunicator(int taskCommIndex) {
- return taskCommunicators[taskCommIndex];
- }
-
- private void pingContainerHeartbeatHandler(ContainerId containerId) {
- containerHeartbeatHandler.pinged(containerId);
- }
-
- private void pingContainerHeartbeatHandler(TezTaskAttemptID taskAttemptId) {
- ContainerId containerId = registeredAttempts.get(taskAttemptId);
- if (containerId != null) {
- containerHeartbeatHandler.pinged(containerId);
- } else {
- LOG.warn("Handling communication from attempt: " + taskAttemptId
- + ", ContainerId not known for this attempt");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
index 9d57ac3..071b008 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java
@@ -47,7 +47,7 @@ public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, Ver
// TODO TEZ-2003 (post) TEZ-2669 Propagate errors baack to the AM with proper error reporting
private final AppContext context;
- private final TaskAttemptListenerImpTezDag taskAttemptListener;
+ private final TaskCommunicatorManager taskCommunicatorManager;
private final int taskCommunicatorIndex;
private final ReentrantReadWriteLock.ReadLock dagChangedReadLock;
private final ReentrantReadWriteLock.WriteLock dagChangedWriteLock;
@@ -56,11 +56,11 @@ public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, Ver
private DAG dag;
public TaskCommunicatorContextImpl(AppContext appContext,
- TaskAttemptListenerImpTezDag taskAttemptListener,
+ TaskCommunicatorManager taskCommunicatorManager,
UserPayload userPayload,
int taskCommunicatorIndex) {
this.context = appContext;
- this.taskAttemptListener = taskAttemptListener;
+ this.taskCommunicatorManager = taskCommunicatorManager;
this.userPayload = userPayload;
this.taskCommunicatorIndex = taskCommunicatorIndex;
@@ -86,13 +86,13 @@ public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, Ver
@Override
public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
- return taskAttemptListener.canCommit(taskAttemptId);
+ return taskCommunicatorManager.canCommit(taskAttemptId);
}
@Override
public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request) throws IOException,
TezException {
- return taskAttemptListener.heartbeat(request);
+ return taskCommunicatorManager.heartbeat(request);
}
@Override
@@ -108,31 +108,31 @@ public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, Ver
@Override
public void taskAlive(TezTaskAttemptID taskAttemptId) {
- taskAttemptListener.taskAlive(taskAttemptId);
+ taskCommunicatorManager.taskAlive(taskAttemptId);
}
@Override
public void containerAlive(ContainerId containerId) {
if (isKnownContainer(containerId)) {
- taskAttemptListener.containerAlive(containerId);
+ taskCommunicatorManager.containerAlive(containerId);
}
}
@Override
public void taskStartedRemotely(TezTaskAttemptID taskAttemptId, ContainerId containerId) {
- taskAttemptListener.taskStartedRemotely(taskAttemptId, containerId);
+ taskCommunicatorManager.taskStartedRemotely(taskAttemptId, containerId);
}
@Override
public void taskKilled(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
@Nullable String diagnostics) {
- taskAttemptListener.taskKilled(taskAttemptId, taskAttemptEndReason, diagnostics);
+ taskCommunicatorManager.taskKilled(taskAttemptId, taskAttemptEndReason, diagnostics);
}
@Override
public void taskFailed(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
@Nullable String diagnostics) {
- taskAttemptListener.taskFailed(taskAttemptId, taskAttemptEndReason, diagnostics);
+ taskCommunicatorManager.taskFailed(taskAttemptId, taskAttemptEndReason, diagnostics);
}
@@ -196,7 +196,7 @@ public class TaskCommunicatorContextImpl implements TaskCommunicatorContext, Ver
@Override
public void onStateUpdated(VertexStateUpdate event) {
try {
- taskAttemptListener.vertexStateUpdateNotificationReceived(event, taskCommunicatorIndex);
+ taskCommunicatorManager.vertexStateUpdateNotificationReceived(event, taskCommunicatorIndex);
} catch (Exception e) {
throw new TezUncheckedException(e);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java
new file mode 100644
index 0000000..42df259
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java
@@ -0,0 +1,449 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.tez.dag.app;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.commons.collections4.ListUtils;
+import org.apache.tez.dag.api.NamedEntityDescriptor;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
+import org.apache.tez.serviceplugins.api.ContainerEndReason;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEvent;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate;
+import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent;
+import org.apache.tez.runtime.api.impl.EventType;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.dag.api.TaskCommunicator;
+import org.apache.tez.dag.api.TaskCommunicatorContext;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.dag.api.TaskHeartbeatResponse;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.dag.api.TaskHeartbeatRequest;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.Task;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptKilled;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
+import org.apache.tez.dag.app.dag.event.VertexEventRouteEvent;
+import org.apache.tez.dag.app.rm.container.AMContainerTask;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.api.impl.TezEvent;
+
+
+@SuppressWarnings("unchecked")
+@InterfaceAudience.Private
+public class TaskCommunicatorManager extends AbstractService implements
+ TaskCommunicatorManagerInterface {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TaskCommunicatorManager.class);
+
+ private final AppContext context;
+ private final TaskCommunicator[] taskCommunicators;
+ private final TaskCommunicatorContext[] taskCommunicatorContexts;
+ protected final ServicePluginLifecycleAbstractService []taskCommunicatorServiceWrappers;
+
+ protected final TaskHeartbeatHandler taskHeartbeatHandler;
+ protected final ContainerHeartbeatHandler containerHeartbeatHandler;
+
+ private final TaskHeartbeatResponse RESPONSE_SHOULD_DIE = new TaskHeartbeatResponse(true, null, 0, 0);
+
+ private final ConcurrentMap<TezTaskAttemptID, ContainerId> registeredAttempts =
+ new ConcurrentHashMap<TezTaskAttemptID, ContainerId>();
+ private final ConcurrentMap<ContainerId, ContainerInfo> registeredContainers =
+ new ConcurrentHashMap<ContainerId, ContainerInfo>();
+
+ // Defined primarily to work around ConcurrentMaps not accepting null values
+ private static final class ContainerInfo {
+ TezTaskAttemptID taskAttemptId;
+ ContainerInfo(TezTaskAttemptID taskAttemptId) {
+ this.taskAttemptId = taskAttemptId;
+ }
+ }
+
+ private static final ContainerInfo NULL_CONTAINER_INFO = new ContainerInfo(null);
+
+
+ public TaskCommunicatorManager(AppContext context,
+ TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
+ List<NamedEntityDescriptor> taskCommunicatorDescriptors) {
+ super(TaskCommunicatorManager.class.getName());
+ this.context = context;
+ this.taskHeartbeatHandler = thh;
+ this.containerHeartbeatHandler = chh;
+ Preconditions.checkArgument(
+ taskCommunicatorDescriptors != null && !taskCommunicatorDescriptors.isEmpty(),
+ "TaskCommunicators must be specified");
+ this.taskCommunicators = new TaskCommunicator[taskCommunicatorDescriptors.size()];
+ this.taskCommunicatorContexts = new TaskCommunicatorContext[taskCommunicatorDescriptors.size()];
+ this.taskCommunicatorServiceWrappers = new ServicePluginLifecycleAbstractService[taskCommunicatorDescriptors.size()];
+ for (int i = 0 ; i < taskCommunicatorDescriptors.size() ; i++) {
+ UserPayload userPayload = taskCommunicatorDescriptors.get(i).getUserPayload();
+ taskCommunicatorContexts[i] = new TaskCommunicatorContextImpl(context, this, userPayload, i);
+ taskCommunicators[i] = createTaskCommunicator(taskCommunicatorDescriptors.get(i), i);
+ taskCommunicatorServiceWrappers[i] = new ServicePluginLifecycleAbstractService(taskCommunicators[i]);
+ }
+ // TODO TEZ-2118 Start using taskCommunicator indices properly
+ }
+
+ @Override
+ public void serviceStart() {
+ // TODO Why is init tied to serviceStart
+ for (int i = 0 ; i < taskCommunicators.length ; i++) {
+ taskCommunicatorServiceWrappers[i].init(getConfig());
+ taskCommunicatorServiceWrappers[i].start();
+ }
+ }
+
+ @Override
+ public void serviceStop() {
+ for (int i = 0 ; i < taskCommunicators.length ; i++) {
+ taskCommunicatorServiceWrappers[i].stop();
+ }
+ }
+
+ @VisibleForTesting
+ TaskCommunicator createTaskCommunicator(NamedEntityDescriptor taskCommDescriptor,
+ int taskCommIndex) {
+ if (taskCommDescriptor.getEntityName().equals(TezConstants.getTezYarnServicePluginName())) {
+ return createDefaultTaskCommunicator(taskCommunicatorContexts[taskCommIndex]);
+ } else if (taskCommDescriptor.getEntityName()
+ .equals(TezConstants.getTezUberServicePluginName())) {
+ return createUberTaskCommunicator(taskCommunicatorContexts[taskCommIndex]);
+ } else {
+ return createCustomTaskCommunicator(taskCommunicatorContexts[taskCommIndex],
+ taskCommDescriptor);
+ }
+ }
+
+ @VisibleForTesting
+ TaskCommunicator createDefaultTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
+ LOG.info("Using Default Task Communicator");
+ return new TezTaskCommunicatorImpl(taskCommunicatorContext);
+ }
+
+ @VisibleForTesting
+ TaskCommunicator createUberTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) {
+ LOG.info("Using Default Local Task Communicator");
+ return new TezLocalTaskCommunicatorImpl(taskCommunicatorContext);
+ }
+
+ @VisibleForTesting
+ TaskCommunicator createCustomTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext,
+ NamedEntityDescriptor taskCommDescriptor) {
+ LOG.info("Using TaskCommunicator {}:{} " + taskCommDescriptor.getEntityName(),
+ taskCommDescriptor.getClassName());
+ Class<? extends TaskCommunicator> taskCommClazz =
+ (Class<? extends TaskCommunicator>) ReflectionUtils
+ .getClazz(taskCommDescriptor.getClassName());
+ try {
+ Constructor<? extends TaskCommunicator> ctor =
+ taskCommClazz.getConstructor(TaskCommunicatorContext.class);
+ return ctor.newInstance(taskCommunicatorContext);
+ } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+
+ public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request)
+ throws IOException, TezException {
+ ContainerId containerId = ConverterUtils.toContainerId(request
+ .getContainerIdentifier());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received heartbeat from container"
+ + ", request=" + request);
+ }
+
+ if (!registeredContainers.containsKey(containerId)) {
+ LOG.warn("Received task heartbeat from unknown container with id: " + containerId +
+ ", asking it to die");
+ return RESPONSE_SHOULD_DIE;
+ }
+
+ // A heartbeat can come in anytime. The AM may have made a decision to kill a running task/container
+ // meanwhile. If the decision is processed through the pipeline before the heartbeat is processed,
+ // the heartbeat will be dropped. Otherwise the heartbeat will be processed - and the system
+ // know how to handle this - via FailedInputEvents for example (relevant only if the heartbeat has events).
+ // So - avoiding synchronization.
+
+ pingContainerHeartbeatHandler(containerId);
+ TaskAttemptEventInfo eventInfo = new TaskAttemptEventInfo(0, null, 0);
+ TezTaskAttemptID taskAttemptID = request.getTaskAttemptId();
+ if (taskAttemptID != null) {
+ ContainerId containerIdFromMap = registeredAttempts.get(taskAttemptID);
+ if (containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
+ // This can happen when a task heartbeats. Meanwhile the container is unregistered.
+ // The information will eventually make it through to the plugin via a corresponding unregister.
+ // There's a race in that case between the unregister making it through, and this method returning.
+ // TODO TEZ-2003 (post) TEZ-2666. An exception back is likely a better approach than sending a shouldDie = true,
+ // so that the plugin can handle the scenario. Alternately augment the response with error codes.
+ // Error codes would be better than exceptions.
+ LOG.info("Attempt: " + taskAttemptID + " is not recognized for heartbeats");
+ return RESPONSE_SHOULD_DIE;
+ }
+
+ List<TezEvent> inEvents = request.getEvents();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ping from " + taskAttemptID.toString() +
+ " events: " + (inEvents != null ? inEvents.size() : -1));
+ }
+
+ long currTime = context.getClock().getTime();
+ List<TezEvent> otherEvents = new ArrayList<TezEvent>();
+ // route TASK_STATUS_UPDATE_EVENT directly to TaskAttempt and route other events
+ // (DATA_MOVEMENT_EVENT, TASK_ATTEMPT_COMPLETED_EVENT, TASK_ATTEMPT_FAILED_EVENT)
+ // to VertexImpl to ensure the events ordering
+ // 1. DataMovementEvent is logged as RecoveryEvent before TaskAttemptFinishedEvent
+ // 2. TaskStatusEvent is handled before TaskAttemptFinishedEvent
+ for (TezEvent tezEvent : ListUtils.emptyIfNull(inEvents)) {
+ // for now, set the event time on the AM when it is received.
+ // this avoids any time disparity between machines.
+ tezEvent.setEventReceivedTime(currTime);
+ final EventType eventType = tezEvent.getEventType();
+ if (eventType == EventType.TASK_STATUS_UPDATE_EVENT) {
+ TaskAttemptEvent taskAttemptEvent = new TaskAttemptEventStatusUpdate(taskAttemptID,
+ (TaskStatusUpdateEvent) tezEvent.getEvent());
+ context.getEventHandler().handle(taskAttemptEvent);
+ } else {
+ otherEvents.add(tezEvent);
+ }
+ }
+ if(!otherEvents.isEmpty()) {
+ TezVertexID vertexId = taskAttemptID.getTaskID().getVertexID();
+ context.getEventHandler().handle(
+ new VertexEventRouteEvent(vertexId, Collections.unmodifiableList(otherEvents)));
+ }
+ taskHeartbeatHandler.pinged(taskAttemptID);
+ eventInfo = context
+ .getCurrentDAG()
+ .getVertex(taskAttemptID.getTaskID().getVertexID())
+ .getTaskAttemptTezEvents(taskAttemptID, request.getStartIndex(), request.getPreRoutedStartIndex(),
+ request.getMaxEvents());
+ }
+ return new TaskHeartbeatResponse(false, eventInfo.getEvents(), eventInfo.getNextFromEventId(), eventInfo.getNextPreRoutedFromEventId());
+ }
+ public void taskAlive(TezTaskAttemptID taskAttemptId) {
+ taskHeartbeatHandler.pinged(taskAttemptId);
+ }
+
+ public void containerAlive(ContainerId containerId) {
+ pingContainerHeartbeatHandler(containerId);
+ }
+
+ public void taskStartedRemotely(TezTaskAttemptID taskAttemptID, ContainerId containerId) {
+ context.getEventHandler()
+ .handle(new TaskAttemptEventStartedRemotely(taskAttemptID, containerId, null));
+ pingContainerHeartbeatHandler(containerId);
+ }
+
+ public void taskKilled(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
+ String diagnostics) {
+ // Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
+ // and messages from the scheduler will release the container.
+ // TODO TEZ-2003 (post) TEZ-2671 Maybe consider un-registering here itself, since the task is not active anymore,
+ // instead of waiting for the unregister to flow through the Container.
+ // Fix along the same lines as TEZ-2124 by introducing an explict context.
+ context.getEventHandler().handle(new TaskAttemptEventAttemptKilled(taskAttemptId,
+ diagnostics, TezUtilsInternal.fromTaskAttemptEndReason(
+ taskAttemptEndReason)));
+ }
+
+ public void taskFailed(TezTaskAttemptID taskAttemptId, TaskAttemptEndReason taskAttemptEndReason,
+ String diagnostics) {
+ // Regular flow via TaskAttempt will take care of un-registering from the heartbeat handler,
+ // and messages from the scheduler will release the container.
+ // TODO TEZ-2003 (post) TEZ-2671 Maybe consider un-registering here itself, since the task is not active anymore,
+ // instead of waiting for the unregister to flow through the Container.
+ // Fix along the same lines as TEZ-2124 by introducing an explict context.
+ context.getEventHandler().handle(new TaskAttemptEventAttemptFailed(taskAttemptId,
+ TaskAttemptEventType.TA_FAILED, diagnostics, TezUtilsInternal.fromTaskAttemptEndReason(
+ taskAttemptEndReason)));
+ }
+
+ public void vertexStateUpdateNotificationReceived(VertexStateUpdate event, int taskCommIndex) throws
+ Exception {
+ taskCommunicators[taskCommIndex].onVertexStateUpdated(event);
+ }
+
+
+ /**
+ * Child checking whether it can commit.
+ * <p/>
+ * <br/>
+ * Repeatedly polls the ApplicationMaster whether it
+ * {@link Task#canCommit(TezTaskAttemptID)} This is * a legacy from the
+ * centralized commit protocol handling by the JobTracker.
+ */
+// @Override
+ public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
+ LOG.info("Commit go/no-go request from " + taskAttemptId.toString());
+ // An attempt is asking if it can commit its output. This can be decided
+ // only by the task which is managing the multiple attempts. So redirect the
+ // request there.
+ taskHeartbeatHandler.progressing(taskAttemptId);
+ pingContainerHeartbeatHandler(taskAttemptId);
+
+ DAG job = context.getCurrentDAG();
+ Task task =
+ job.getVertex(taskAttemptId.getTaskID().getVertexID()).
+ getTask(taskAttemptId.getTaskID());
+ return task.canCommit(taskAttemptId);
+ }
+
+ // The TaskAttemptListener register / unregister methods in this class are not thread safe.
+ // The Tez framework should not invoke these methods from multiple threads.
+ @Override
+ public void dagComplete(DAG dag) {
+ // TODO TEZ-2335. Cleanup TaskHeartbeat handler structures.
+ // TODO TEZ-2345. Also cleanup attemptInfo map, so that any tasks which heartbeat are told to die.
+ // Container structures remain unchanged - since they could be re-used across restarts.
+ // This becomes more relevant when task kills without container kills are allowed.
+
+ // TODO TEZ-2336. Send a signal to containers indicating DAG completion.
+
+ // Inform all communicators of the dagCompletion.
+ for (int i = 0 ; i < taskCommunicators.length ; i++) {
+ ((TaskCommunicatorContextImpl)taskCommunicatorContexts[i]).dagCompleteStart(dag);
+ taskCommunicators[i].dagComplete(dag.getName());
+ ((TaskCommunicatorContextImpl)taskCommunicatorContexts[i]).dagCompleteEnd();
+ }
+
+ }
+
+ @Override
+ public void dagSubmitted() {
+ // Nothing to do right now. Indicates that a new DAG has been submitted and
+ // the context has updated information.
+ }
+
+ @Override
+ public void registerRunningContainer(ContainerId containerId, int taskCommId) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ContainerId: " + containerId + " registered with TaskAttemptListener");
+ }
+ ContainerInfo oldInfo = registeredContainers.put(containerId, NULL_CONTAINER_INFO);
+ if (oldInfo != null) {
+ throw new TezUncheckedException(
+ "Multiple registrations for containerId: " + containerId);
+ }
+ NodeId nodeId = context.getAllContainers().get(containerId).getContainer().getNodeId();
+ taskCommunicators[taskCommId].registerRunningContainer(containerId, nodeId.getHost(),
+ nodeId.getPort());
+ }
+
+ @Override
+ public void unregisterRunningContainer(ContainerId containerId, int taskCommId, ContainerEndReason endReason, String diagnostics) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unregistering Container from TaskAttemptListener: " + containerId);
+ }
+ ContainerInfo containerInfo = registeredContainers.remove(containerId);
+ if (containerInfo.taskAttemptId != null) {
+ registeredAttempts.remove(containerInfo.taskAttemptId);
+ }
+ taskCommunicators[taskCommId].registerContainerEnd(containerId, endReason, diagnostics);
+ }
+
+ @Override
+ public void registerTaskAttempt(AMContainerTask amContainerTask,
+ ContainerId containerId, int taskCommId) {
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ if (containerInfo == null) {
+ throw new TezUncheckedException("Registering task attempt: "
+ + amContainerTask.getTask().getTaskAttemptID() + " to unknown container: " + containerId);
+ }
+ if (containerInfo.taskAttemptId != null) {
+ throw new TezUncheckedException("Registering task attempt: "
+ + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+ + " with existing assignment to: " +
+ containerInfo.taskAttemptId);
+ }
+
+ // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
+ registeredContainers.put(containerId, new ContainerInfo(amContainerTask.getTask().getTaskAttemptID()));
+
+ ContainerId containerIdFromMap = registeredAttempts.put(
+ amContainerTask.getTask().getTaskAttemptID(), containerId);
+ if (containerIdFromMap != null) {
+ throw new TezUncheckedException("Registering task attempt: "
+ + amContainerTask.getTask().getTaskAttemptID() + " to container: " + containerId
+ + " when already assigned to: " + containerIdFromMap);
+ }
+ taskCommunicators[taskCommId].registerRunningTaskAttempt(containerId, amContainerTask.getTask(),
+ amContainerTask.getAdditionalResources(), amContainerTask.getCredentials(),
+ amContainerTask.haveCredentialsChanged(), amContainerTask.getPriority());
+ }
+
+ @Override
+ public void unregisterTaskAttempt(TezTaskAttemptID attemptId, int taskCommId, TaskAttemptEndReason endReason, String diagnostics) {
+ ContainerId containerId = registeredAttempts.remove(attemptId);
+ if (containerId == null) {
+ LOG.warn("Unregister task attempt: " + attemptId + " from unknown container");
+ return;
+ }
+ ContainerInfo containerInfo = registeredContainers.get(containerId);
+ if (containerInfo == null) {
+ LOG.warn("Unregister task attempt: " + attemptId +
+ " from non-registered container: " + containerId);
+ return;
+ }
+ // Explicitly putting in a new entry so that synchronization is not required on the existing element in the map.
+ registeredContainers.put(containerId, NULL_CONTAINER_INFO);
+ taskCommunicators[taskCommId].unregisterRunningTaskAttempt(attemptId, endReason, diagnostics);
+ }
+
+ @Override
+ public TaskCommunicator getTaskCommunicator(int taskCommIndex) {
+ return taskCommunicators[taskCommIndex];
+ }
+
+ private void pingContainerHeartbeatHandler(ContainerId containerId) {
+ containerHeartbeatHandler.pinged(containerId);
+ }
+
+ private void pingContainerHeartbeatHandler(TezTaskAttemptID taskAttemptId) {
+ ContainerId containerId = registeredAttempts.get(taskAttemptId);
+ if (containerId != null) {
+ containerHeartbeatHandler.pinged(containerId);
+ } else {
+ LOG.warn("Handling communication from attempt: " + taskAttemptId
+ + ", ContainerId not known for this attempt");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java
new file mode 100644
index 0000000..8d060a2
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java
@@ -0,0 +1,46 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.app;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.serviceplugins.api.ContainerEndReason;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.api.TaskCommunicator;
+import org.apache.tez.dag.app.rm.container.AMContainerTask;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+/**
+ * This class listens for changes to the state of a Task.
+ */
+public interface TaskCommunicatorManagerInterface {
+
+ void registerRunningContainer(ContainerId containerId, int taskCommId);
+
+ void registerTaskAttempt(AMContainerTask amContainerTask, ContainerId containerId, int taskCommId);
+
+ void unregisterRunningContainer(ContainerId containerId, int taskCommId, ContainerEndReason endReason, String diagnostics);
+
+ void unregisterTaskAttempt(TezTaskAttemptID attemptID, int taskCommId, TaskAttemptEndReason endReason, String diagnostics);
+
+ void dagComplete(DAG dag);
+
+ void dagSubmitted();
+
+ TaskCommunicator getTaskCommunicator(int taskCommIndex);
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 6b474ff..756ed28 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -82,7 +82,7 @@ import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
import org.apache.tez.dag.api.records.DAGProtos.PlanVertexGroupInfo;
import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.app.AppContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.DAGReport;
@@ -160,7 +160,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private final Lock readLock;
private final Lock writeLock;
private final String dagName;
- private final TaskAttemptListener taskAttemptListener;
+ private final TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
private final TaskHeartbeatHandler taskHeartbeatHandler;
private final Object tasksSyncHandle = new Object();
@@ -489,7 +489,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
Configuration amConf,
DAGPlan jobPlan,
EventHandler eventHandler,
- TaskAttemptListener taskAttemptListener,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
Credentials dagCredentials,
Clock clock,
String appUserName,
@@ -511,7 +511,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
this.clock = clock;
this.appContext = appContext;
- this.taskAttemptListener = taskAttemptListener;
+ this.taskCommunicatorManagerInterface = taskCommunicatorManagerInterface;
this.taskHeartbeatHandler = thh;
this.eventHandler = eventHandler;
ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
@@ -1538,7 +1538,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
VertexImpl v = new VertexImpl(
vertexId, vertexPlan, vertexName, dag.dagConf,
- dag.eventHandler, dag.taskAttemptListener,
+ dag.eventHandler, dag.taskCommunicatorManagerInterface,
dag.clock, dag.taskHeartbeatHandler,
!dag.commitAllOutputsOnSuccess, dag.appContext, vertexLocationHint,
dag.vertexGroups, dag.taskSpecificLaunchCmdOption, dag.entityUpdateTracker);
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 6957b1d..3f2e3a4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -61,7 +61,7 @@ import org.apache.tez.dag.api.oldrecords.TaskAttemptReport;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttempt;
@@ -403,17 +403,17 @@ public class TaskAttemptImpl implements TaskAttempt,
@SuppressWarnings("rawtypes")
public TaskAttemptImpl(TezTaskID taskId, int attemptNumber, EventHandler eventHandler,
- TaskAttemptListener taskAttemptListener, Configuration conf, Clock clock,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock,
TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
boolean isRescheduled,
Resource resource, ContainerContext containerContext, boolean leafVertex,
Task task) {
- this(taskId, attemptNumber, eventHandler, taskAttemptListener, conf, clock,
+ this(taskId, attemptNumber, eventHandler, taskCommunicatorManagerInterface, conf, clock,
taskHeartbeatHandler, appContext, isRescheduled, resource, containerContext, leafVertex,
task, null);
}
public TaskAttemptImpl(TezTaskID taskId, int attemptNumber, EventHandler eventHandler,
- TaskAttemptListener taskAttemptListener, Configuration conf, Clock clock,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock,
TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
boolean isRescheduled,
Resource resource, ContainerContext containerContext, boolean leafVertex,
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 1b55295..ea14483 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -56,7 +56,7 @@ import org.apache.tez.dag.api.oldrecords.TaskReport;
import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerContext;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.StateChangeNotifier;
import org.apache.tez.dag.app.dag.Task;
@@ -115,7 +115,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
.getProperty("line.separator");
protected final Configuration conf;
- protected final TaskAttemptListener taskAttemptListener;
+ protected final TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
protected final TaskHeartbeatHandler taskHeartbeatHandler;
protected final EventHandler eventHandler;
private final TezTaskID taskId;
@@ -341,7 +341,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
public TaskImpl(TezVertexID vertexId, int taskIndex,
EventHandler eventHandler, Configuration conf,
- TaskAttemptListener taskAttemptListener,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
Clock clock, TaskHeartbeatHandler thh, AppContext appContext,
boolean leafVertex, Resource resource,
ContainerContext containerContext,
@@ -357,7 +357,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
maxFailedAttempts = this.conf.getInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS,
TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS_DEFAULT);
taskId = TezTaskID.getInstance(vertexId, taskIndex);
- this.taskAttemptListener = taskAttemptListener;
+ this.taskCommunicatorManagerInterface = taskCommunicatorManagerInterface;
this.taskHeartbeatHandler = thh;
this.eventHandler = eventHandler;
this.appContext = appContext;
@@ -817,7 +817,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
TaskAttemptImpl createAttempt(int attemptNumber, TezTaskAttemptID schedulingCausalTA) {
return new TaskAttemptImpl(getTaskId(), attemptNumber, eventHandler,
- taskAttemptListener, conf, clock, taskHeartbeatHandler, appContext,
+ taskCommunicatorManagerInterface, conf, clock, taskHeartbeatHandler, appContext,
(failedAttempts > 0), taskResource, containerContext, leafVertex, this, schedulingCausalTA);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/8b278ea8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 3cc439f..a1dcf6c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -94,7 +94,7 @@ import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerContext;
import org.apache.tez.dag.app.TaskAttemptEventInfo;
-import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskCommunicatorManagerInterface;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.RootInputInitializerManager;
@@ -212,7 +212,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl
private final Lock readLock;
private final Lock writeLock;
- private final TaskAttemptListener taskAttemptListener;
+ private final TaskCommunicatorManagerInterface taskCommunicatorManagerInterface;
private final TaskHeartbeatHandler taskHeartbeatHandler;
private final Object tasksSyncHandle = new Object();
@@ -890,7 +890,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl
public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan,
String vertexName, Configuration dagConf, EventHandler eventHandler,
- TaskAttemptListener taskAttemptListener, Clock clock,
+ TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Clock clock,
TaskHeartbeatHandler thh, boolean commitVertexOutputs,
AppContext appContext, VertexLocationHint vertexLocationHint,
Map<String, VertexGroupInfo> dagVertexGroups, TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption,
@@ -911,7 +911,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl
this.appContext = appContext;
this.commitVertexOutputs = commitVertexOutputs;
- this.taskAttemptListener = taskAttemptListener;
+ this.taskCommunicatorManagerInterface = taskCommunicatorManagerInterface;
this.taskHeartbeatHandler = thh;
this.eventHandler = eventHandler;
ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
@@ -2331,7 +2331,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl
return new TaskImpl(this.getVertexId(), taskIndex,
this.eventHandler,
vertexConf,
- this.taskAttemptListener,
+ this.taskCommunicatorManagerInterface,
this.clock,
this.taskHeartbeatHandler,
this.appContext,