You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2013/06/04 07:33:03 UTC
[01/50] [abbrv] git commit: TEZ-121: build break due to
MAPREDUCE-4067 (protocol exception types)
Updated Branches:
refs/heads/master d6cd24bbe -> 809bbd964
TEZ-121: build break due to MAPREDUCE-4067 (protocol exception types)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/97e1fa96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/97e1fa96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/97e1fa96
Branch: refs/heads/master
Commit: 97e1fa9671b3f2b04c5d2ef7547bb882da351ea7
Parents: c86c279
Author: mikelid <mi...@microsoft.com>
Authored: Mon May 13 11:16:09 2013 -0700
Committer: mikelid <mi...@microsoft.com>
Committed: Mon May 13 11:16:09 2013 -0700
----------------------------------------------------------------------
.../org/apache/tez/mapreduce/NotRunningJob.java | 30 +++++++-------
1 files changed, 15 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/97e1fa96/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
index 7199edf..370c8f0 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
@@ -18,6 +18,7 @@
package org.apache.tez.mapreduce;
+import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
@@ -65,7 +66,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -101,7 +101,7 @@ public class NotRunningJob implements MRClientProtocol {
@Override
public FailTaskAttemptResponse failTaskAttempt(
- FailTaskAttemptRequest request) throws YarnRemoteException {
+ FailTaskAttemptRequest request) throws IOException {
FailTaskAttemptResponse resp =
recordFactory.newRecordInstance(FailTaskAttemptResponse.class);
return resp;
@@ -109,7 +109,7 @@ public class NotRunningJob implements MRClientProtocol {
@Override
public GetCountersResponse getCounters(GetCountersRequest request)
- throws YarnRemoteException {
+ throws IOException {
GetCountersResponse resp =
recordFactory.newRecordInstance(GetCountersResponse.class);
Counters counters = recordFactory.newRecordInstance(Counters.class);
@@ -120,7 +120,7 @@ public class NotRunningJob implements MRClientProtocol {
@Override
public GetDiagnosticsResponse getDiagnostics(GetDiagnosticsRequest request)
- throws YarnRemoteException {
+ throws IOException {
GetDiagnosticsResponse resp =
recordFactory.newRecordInstance(GetDiagnosticsResponse.class);
resp.addDiagnostics("");
@@ -129,7 +129,7 @@ public class NotRunningJob implements MRClientProtocol {
@Override
public GetJobReportResponse getJobReport(GetJobReportRequest request)
- throws YarnRemoteException {
+ throws IOException {
JobReport jobReport =
recordFactory.newRecordInstance(JobReport.class);
jobReport.setJobId(request.getJobId());
@@ -150,7 +150,7 @@ public class NotRunningJob implements MRClientProtocol {
@Override
public GetTaskAttemptCompletionEventsResponse getTaskAttemptCompletionEvents(
GetTaskAttemptCompletionEventsRequest request)
- throws YarnRemoteException {
+ throws IOException {
GetTaskAttemptCompletionEventsResponse resp =
recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class);
resp.addAllCompletionEvents(new ArrayList<TaskAttemptCompletionEvent>());
@@ -159,14 +159,14 @@ public class NotRunningJob implements MRClientProtocol {
@Override
public GetTaskAttemptReportResponse getTaskAttemptReport(
- GetTaskAttemptReportRequest request) throws YarnRemoteException {
+ GetTaskAttemptReportRequest request) throws IOException {
//not invoked by anybody
throw new NotImplementedException();
}
@Override
public GetTaskReportResponse getTaskReport(GetTaskReportRequest request)
- throws YarnRemoteException {
+ throws IOException {
GetTaskReportResponse resp =
recordFactory.newRecordInstance(GetTaskReportResponse.class);
TaskReport report = recordFactory.newRecordInstance(TaskReport.class);
@@ -181,7 +181,7 @@ public class NotRunningJob implements MRClientProtocol {
@Override
public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request)
- throws YarnRemoteException {
+ throws IOException {
GetTaskReportsResponse resp =
recordFactory.newRecordInstance(GetTaskReportsResponse.class);
resp.addAllTaskReports(new ArrayList<TaskReport>());
@@ -190,7 +190,7 @@ public class NotRunningJob implements MRClientProtocol {
@Override
public KillJobResponse killJob(KillJobRequest request)
- throws YarnRemoteException {
+ throws IOException {
KillJobResponse resp =
recordFactory.newRecordInstance(KillJobResponse.class);
return resp;
@@ -198,7 +198,7 @@ public class NotRunningJob implements MRClientProtocol {
@Override
public KillTaskResponse killTask(KillTaskRequest request)
- throws YarnRemoteException {
+ throws IOException {
KillTaskResponse resp =
recordFactory.newRecordInstance(KillTaskResponse.class);
return resp;
@@ -206,7 +206,7 @@ public class NotRunningJob implements MRClientProtocol {
@Override
public KillTaskAttemptResponse killTaskAttempt(
- KillTaskAttemptRequest request) throws YarnRemoteException {
+ KillTaskAttemptRequest request) throws IOException {
KillTaskAttemptResponse resp =
recordFactory.newRecordInstance(KillTaskAttemptResponse.class);
return resp;
@@ -214,21 +214,21 @@ public class NotRunningJob implements MRClientProtocol {
@Override
public GetDelegationTokenResponse getDelegationToken(
- GetDelegationTokenRequest request) throws YarnRemoteException {
+ GetDelegationTokenRequest request) throws IOException {
/* Should not be invoked by anyone. */
throw new NotImplementedException();
}
@Override
public RenewDelegationTokenResponse renewDelegationToken(
- RenewDelegationTokenRequest request) throws YarnRemoteException {
+ RenewDelegationTokenRequest request) throws IOException {
/* Should not be invoked by anyone. */
throw new NotImplementedException();
}
@Override
public CancelDelegationTokenResponse cancelDelegationToken(
- CancelDelegationTokenRequest request) throws YarnRemoteException {
+ CancelDelegationTokenRequest request) throws IOException {
/* Should not be invoked by anyone. */
throw new NotImplementedException();
}
[40/50] [abbrv] git commit: TEZ-174. Fix hadoop-maven-plugin version
should be managed through pluginManagement
Posted by ss...@apache.org.
TEZ-174. Fix hadoop-maven-plugin version should be managed through pluginManagement
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/d3b19212
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/d3b19212
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/d3b19212
Branch: refs/heads/master
Commit: d3b19212c7cd01aa26c31ad0eb40d50662144f54
Parents: a679189
Author: Giridharan Kesavan <gk...@apache.org>
Authored: Fri May 31 06:18:45 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri May 31 07:15:53 2013 -0700
----------------------------------------------------------------------
pom.xml | 5 +++++
tez-dag-api/pom.xml | 1 -
2 files changed, 5 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d3b19212/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0e4f8e7..721df2f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -301,6 +301,11 @@
<version>2.3.1</version>
</plugin>
<plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <version>${hadoop.version}</version>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d3b19212/tez-dag-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag-api/pom.xml b/tez-dag-api/pom.xml
index 3cb0952..2a91b4d 100644
--- a/tez-dag-api/pom.xml
+++ b/tez-dag-api/pom.xml
@@ -64,7 +64,6 @@
<plugin>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
- <version>3.0.0-SNAPSHOT</version>
<executions>
<execution>
<id>compile-protoc</id>
[10/50] [abbrv] git commit: TEZ-128. Add additional unit tests for
TaskAttempt.
Posted by ss...@apache.org.
TEZ-128. Add additional unit tests for TaskAttempt.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/d660948d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/d660948d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/d660948d
Branch: refs/heads/master
Commit: d660948d7eaa3536a3bcd97e16524aa626b4ad7c
Parents: 4c89e15
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue May 14 15:31:07 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue May 14 15:31:07 2013 -0700
----------------------------------------------------------------------
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 5 -
.../tez/dag/app/dag/impl/TestTaskAttempt.java | 409 ++++++++++++---
.../apache/tez/dag/app/dag/impl/TestTaskImpl.java | 2 +
3 files changed, 325 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d660948d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 2564ffa..264b3ad 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -116,7 +116,6 @@ public class TaskAttemptImpl implements TaskAttempt,
protected EventHandler eventHandler;
private final TezTaskAttemptID attemptId;
private final Clock clock;
-// private final TaskAttemptListener taskAttemptListener;
private final List<String> diagnostics = new ArrayList<String>();
private final Lock readLock;
private final Lock writeLock;
@@ -923,10 +922,6 @@ public class TaskAttemptImpl implements TaskAttempt,
// Send out events to the Task - indicating TaskAttemptTermination(F/K)
ta.sendEvent(new TaskEventTAUpdate(ta.attemptId, helper
.getTaskEventType()));
-
- if (event instanceof DiagnosableEvent) {
- ta.addDiagnosticInfo(((DiagnosableEvent) event).getDiagnosticInfo());
- }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d660948d/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index a18bd3b..369bced 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -20,13 +20,18 @@ package org.apache.tez.dag.app.dag.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.hadoop.fs.FileStatus;
@@ -61,10 +66,14 @@ import org.apache.tez.dag.app.dag.event.DAGEvent;
import org.apache.tez.dag.app.dag.event.DAGEventType;
import org.apache.tez.dag.app.dag.event.TaskAttemptEvent;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminating;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventKillRequest;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
+import org.apache.tez.dag.app.dag.event.TaskEventTAUpdate;
+import org.apache.tez.dag.app.rm.AMSchedulerEventTAEnded;
+import org.apache.tez.dag.app.rm.AMSchedulerEventTALaunchRequest;
import org.apache.tez.dag.app.rm.container.AMContainerMap;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -72,6 +81,7 @@ import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
@SuppressWarnings({ "unchecked", "rawtypes" })
public class TestTaskAttempt {
@@ -102,88 +112,96 @@ public class TestTaskAttempt {
// testMRAppHistory(app);
// }
- // @Test
- // // Verifies that the launch request is based on the hosts.
- // // TODO Move to the client.
- // // TODO Add a test that verifies that the LocationHint is used as it should
- // be.
- // public void testSingleRackRequest() throws Exception {
- // TaskAttemptImpl.ScheduleTaskattemptTransition sta =
- // new TaskAttemptImpl.ScheduleTaskattemptTransition();
- //
- // EventHandler eventHandler = mock(EventHandler.class);
- // String[] hosts = new String[3];
- // hosts[0] = "host1";
- // hosts[1] = "host2";
- // hosts[2] = "host3";
- // TaskSplitMetaInfo splitInfo = new TaskSplitMetaInfo(hosts, 0,
- // 128 * 1024 * 1024l);
- //
- // TaskAttemptImpl mockTaskAttempt = createMapTaskAttemptImpl2ForTest(
- // eventHandler, splitInfo);
- // TaskAttemptEventSchedule mockTAEvent =
- // mock(TaskAttemptEventSchedule.class);
- // doReturn(false).when(mockTAEvent).isRescheduled();
- //
- // sta.transition(mockTaskAttempt, mockTAEvent);
- //
- // ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
- // verify(eventHandler, times(2)).handle(arg.capture());
- // if (!(arg.getAllValues().get(1) instanceof
- // AMSchedulerTALaunchRequestEvent)) {
- // Assert.fail("Second Event not of type ContainerRequestEvent");
- // }
- // AMSchedulerTALaunchRequestEvent tlrE = (AMSchedulerTALaunchRequestEvent)
- // arg
- // .getAllValues().get(1);
- // String[] requestedRacks = tlrE.getRacks();
- // // Only a single occurrence of /DefaultRack
- // assertEquals(1, requestedRacks.length);
- // }
+ @Test
+ public void testLocalityRequest() {
- // @Test
- // // Tests that an attempt is made to resolve the localized hosts to racks.
- // // TODO Move to the client.
- // public void testHostResolveAttempt() throws Exception {
- // TaskAttemptImpl.ScheduleTaskattemptTransition sta =
- // new TaskAttemptImpl.ScheduleTaskattemptTransition();
- //
- // EventHandler eventHandler = mock(EventHandler.class);
- // String hosts[] = new String[] {"192.168.1.1", "host2", "host3"};
- // String resolved[] = new String[] {"host1", "host2", "host3"};
- // TaskSplitMetaInfo splitInfo =
- // new TaskSplitMetaInfo(hosts, 0, 128 * 1024 * 1024l);
- //
- // TaskAttemptImpl mockTaskAttempt =
- // createMapTaskAttemptImpl2ForTest(eventHandler, splitInfo);
- // TaskAttemptImpl spyTa = spy(mockTaskAttempt);
- // when(spyTa.resolveHosts(hosts)).thenReturn(resolved);
- //
- // TaskAttemptEventSchedule mockTAEvent =
- // mock(TaskAttemptEventSchedule.class);
- // doReturn(false).when(mockTAEvent).isRescheduled();
- //
- // sta.transition(spyTa, mockTAEvent);
- // verify(spyTa).resolveHosts(hosts);
- // ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
- // verify(eventHandler, times(2)).handle(arg.capture());
- // if (!(arg.getAllValues().get(1) instanceof
- // AMSchedulerTALaunchRequestEvent)) {
- // Assert.fail("Second Event not of type ContainerRequestEvent");
- // }
- // Map<String, Boolean> expected = new HashMap<String, Boolean>();
- // expected.put("host1", true);
- // expected.put("host2", true);
- // expected.put("host3", true);
- // AMSchedulerTALaunchRequestEvent cre =
- // (AMSchedulerTALaunchRequestEvent) arg.getAllValues().get(1);
- // String[] requestedHosts = cre.getHosts();
- // for (String h : requestedHosts) {
- // expected.remove(h);
- // }
- // assertEquals(0, expected.size());
- // }
- //
+ TaskAttemptImpl.ScheduleTaskattemptTransition sta =
+ new TaskAttemptImpl.ScheduleTaskattemptTransition();
+
+ EventHandler eventHandler = mock(EventHandler.class);
+ String[] hosts = new String[3];
+ hosts[0] = "host1";
+ hosts[1] = "host2";
+ hosts[2] = "host3";
+ TaskLocationHint locationHint = new TaskLocationHint(hosts, null);
+
+ TezTaskID taskID = new TezTaskID(
+ new TezVertexID(new TezDAGID("1", 1, 1), 1), 1);
+ TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
+ mock(TaskAttemptListener.class), 1, new TezConfiguration(),
+ mock(Token.class), new Credentials(), new SystemClock(),
+ mock(TaskHeartbeatHandler.class), mock(AppContext.class),
+ MAP_PROCESSOR_NAME, locationHint, BuilderUtils.newResource(1024, 1),
+ new HashMap<String, LocalResource>(), new HashMap<String, String>(),
+ "", false);
+
+ TaskAttemptEventSchedule sEvent = mock(TaskAttemptEventSchedule.class);
+
+ sta.transition(taImpl, sEvent);
+
+ ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(1)).handle(arg.capture());
+ if (!(arg.getAllValues().get(0) instanceof AMSchedulerEventTALaunchRequest)) {
+ fail("Second event not of type "
+ + AMSchedulerEventTALaunchRequest.class.getName());
+ }
+ // TODO Move the Rack request check to the client after TEZ-125 is fixed.
+ AMSchedulerEventTALaunchRequest lre = (AMSchedulerEventTALaunchRequest) arg
+ .getAllValues().get(0);
+ String[] requestedRacks = lre.getRacks();
+ assertEquals(1, requestedRacks.length);
+ assertEquals(3, lre.getHosts().length);
+ for (int i = 0; i < 3; i++) {
+ assertEquals("host" + (i + 1), lre.getHosts()[i]);
+ }
+ }
+
+
+ @Test
+ // Tests that an attempt is made to resolve the localized hosts to racks.
+ // TODO Move to the client post TEZ-125.
+ public void testHostResolveAttempt() throws Exception {
+ TaskAttemptImpl.ScheduleTaskattemptTransition sta =
+ new TaskAttemptImpl.ScheduleTaskattemptTransition();
+
+ EventHandler eventHandler = mock(EventHandler.class);
+ String hosts[] = new String[] { "192.168.1.1", "host2", "host3" };
+ String resolved[] = new String[] { "host1", "host2", "host3" };
+ TaskLocationHint locationHint = new TaskLocationHint(hosts, null);
+
+ TezTaskID taskID = new TezTaskID(
+ new TezVertexID(new TezDAGID("1", 1, 1), 1), 1);
+ TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
+ mock(TaskAttemptListener.class), 1, new TezConfiguration(),
+ mock(Token.class), new Credentials(), new SystemClock(),
+ mock(TaskHeartbeatHandler.class), mock(AppContext.class),
+ MAP_PROCESSOR_NAME, locationHint, BuilderUtils.newResource(1024, 1),
+ new HashMap<String, LocalResource>(), new HashMap<String, String>(),
+ "", false);
+ TaskAttemptImpl spyTa = spy(taImpl);
+ when(spyTa.resolveHosts(hosts)).thenReturn(resolved);
+
+ TaskAttemptEventSchedule mockTAEvent = mock(TaskAttemptEventSchedule.class);
+
+ sta.transition(spyTa, mockTAEvent);
+ verify(spyTa).resolveHosts(hosts);
+ ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(1)).handle(arg.capture());
+ if (!(arg.getAllValues().get(0) instanceof AMSchedulerEventTALaunchRequest)) {
+ fail("Second Event not of type ContainerRequestEvent");
+ }
+ Map<String, Boolean> expected = new HashMap<String, Boolean>();
+ expected.put("host1", true);
+ expected.put("host2", true);
+ expected.put("host3", true);
+ AMSchedulerEventTALaunchRequest cre = (AMSchedulerEventTALaunchRequest) arg
+ .getAllValues().get(0);
+ String[] requestedHosts = cre.getHosts();
+ for (String h : requestedHosts) {
+ expected.remove(h);
+ }
+ assertEquals(0, expected.size());
+ }
// @Test
// // Verifies accounting of slot_milli counters. Time spent in running tasks.
@@ -325,10 +343,105 @@ public class TestTaskAttempt {
// null));
assertFalse(eventHandler.internalError);
}
+
+ @Test
+ // Ensure ContainerTerminating and ContainerTerminated is handled correctly by
+ // the TaskAttempt
+ public void testContainerTerminationWhileRunning() throws Exception {
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
+ ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ appId, 0);
+ TezDAGID dagID = new TezDAGID(appId, 1);
+ TezVertexID vertexID = new TezVertexID(dagID, 1);
+ TezTaskID taskID = new TezTaskID(vertexID, 1);
+ TezTaskAttemptID taskAttemptID = new TezTaskAttemptID(taskID, 0);
- // TODO Add a similar test for TERMINATING.
- // Ensure ContainerTerminated is handled correctly by the TaskAttempt
+ MockEventHandler eventHandler = spy(new MockEventHandler());
+ TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+ when(taListener.getAddress()).thenReturn(
+ new InetSocketAddress("localhost", 0));
+
+ TezConfiguration tezConf = new TezConfiguration();
+ tezConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+ tezConf.setBoolean("fs.file.impl.disable.cache", true);
+
+ TaskLocationHint locationHint = new TaskLocationHint(
+ new String[] { "127.0.0.1" }, null);
+ Resource resource = BuilderUtils.newResource(1024, 1);
+ Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+ Map<String, String> environment = new HashMap<String, String>();
+ String javaOpts = "";
+
+ NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
+ ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ Container container = mock(Container.class);
+ when(container.getId()).thenReturn(contId);
+ when(container.getNodeId()).thenReturn(nid);
+ when(container.getNodeHttpAddress()).thenReturn("localhost:0");
+
+ AppContext appCtx = mock(AppContext.class);
+ AMContainerMap containers = new AMContainerMap(
+ mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ appCtx);
+ containers.addContainerIfNew(container);
+
+ doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
+ doReturn(containers).when(appCtx).getAllContainers();
+
+ TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
+ taListener, 1, tezConf, mock(Token.class), new Credentials(),
+ new SystemClock(), mock(TaskHeartbeatHandler.class), appCtx,
+ MAP_PROCESSOR_NAME, locationHint, resource, localResources,
+ environment, javaOpts, false);
+
+ ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+
+ taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, null));
+ // At state STARTING.
+ taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID, contId,
+ null, -1));
+ assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(),
+ TaskAttemptState.RUNNING);
+
+ int expectedEventsAtRunning = 3;
+ verify(eventHandler, times(expectedEventsAtRunning)).handle(arg.capture());
+
+ taImpl.handle(new TaskAttemptEventContainerTerminating(taskAttemptID,
+ "Terminating"));
+ assertFalse(
+ "InternalError occurred trying to handle TA_CONTAINER_TERMINATING",
+ eventHandler.internalError);
+
+ assertEquals("Task attempt is not in the FAILED state", taImpl.getState(),
+ TaskAttemptState.FAILED);
+
+ assertEquals(1, taImpl.getDiagnostics().size());
+ assertEquals("Terminating", taImpl.getDiagnostics().get(0));
+
+ int expectedEvenstAfterTerminating = expectedEventsAtRunning + 3;
+ arg = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture());
+
+ verifyEventType(
+ arg.getAllValues().subList(expectedEventsAtRunning,
+ expectedEvenstAfterTerminating), TaskEventTAUpdate.class, 1);
+ verifyEventType(
+ arg.getAllValues().subList(expectedEventsAtRunning,
+ expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1);
+
+ taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID,
+ "Terminated"));
+ int expectedEventAfterTerminated = expectedEvenstAfterTerminating + 0;
+ arg = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(expectedEventAfterTerminated)).handle(arg.capture());
+
+ assertEquals(2, taImpl.getDiagnostics().size());
+ assertEquals("Terminated", taImpl.getDiagnostics().get(1));
+ }
+
+
@Test
+ // Ensure ContainerTerminated is handled correctly by the TaskAttempt
public void testContainerTerminatedWhileRunning() throws Exception {
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
@@ -382,11 +495,14 @@ public class TestTaskAttempt {
null, -1));
assertEquals("Task attempt is not in running state", taImpl.getState(),
TaskAttemptState.RUNNING);
- taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID, null));
+ taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID, "Terminated"));
assertFalse(
"InternalError occurred trying to handle TA_CONTAINER_TERMINATED",
eventHandler.internalError);
- // TODO Verify diagnostics
+
+ assertEquals("Terminated", taImpl.getDiagnostics().get(0));
+
+ // TODO Ensure TA_TERMINATING after this is ingored.
}
@Test
@@ -456,6 +572,98 @@ public class TestTaskAttempt {
}
@Test
+ // Ensure ContainerTerminating and ContainerTerminated is handled correctly by
+ // the TaskAttempt
+ public void testContainerTerminatedAfterSuccess() throws Exception {
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
+ ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ appId, 0);
+ TezDAGID dagID = new TezDAGID(appId, 1);
+ TezVertexID vertexID = new TezVertexID(dagID, 1);
+ TezTaskID taskID = new TezTaskID(vertexID, 1);
+ TezTaskAttemptID taskAttemptID = new TezTaskAttemptID(taskID, 0);
+
+ MockEventHandler eventHandler = spy(new MockEventHandler());
+ TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+ when(taListener.getAddress()).thenReturn(
+ new InetSocketAddress("localhost", 0));
+
+ TezConfiguration tezConf = new TezConfiguration();
+ tezConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+ tezConf.setBoolean("fs.file.impl.disable.cache", true);
+
+ TaskLocationHint locationHint = new TaskLocationHint(
+ new String[] { "127.0.0.1" }, null);
+ Resource resource = BuilderUtils.newResource(1024, 1);
+ Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+ Map<String, String> environment = new HashMap<String, String>();
+ String javaOpts = "";
+
+ NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
+ ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ Container container = mock(Container.class);
+ when(container.getId()).thenReturn(contId);
+ when(container.getNodeId()).thenReturn(nid);
+ when(container.getNodeHttpAddress()).thenReturn("localhost:0");
+
+ AppContext appCtx = mock(AppContext.class);
+ AMContainerMap containers = new AMContainerMap(
+ mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
+ appCtx);
+ containers.addContainerIfNew(container);
+
+ doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
+ doReturn(containers).when(appCtx).getAllContainers();
+
+ TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
+ taListener, 1, tezConf, mock(Token.class), new Credentials(),
+ new SystemClock(), mock(TaskHeartbeatHandler.class), appCtx,
+ MAP_PROCESSOR_NAME, locationHint, resource, localResources,
+ environment, javaOpts, false);
+
+ ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+
+ taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, null));
+ // At state STARTING.
+ taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID, contId,
+ null, -1));
+ assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(),
+ TaskAttemptState.RUNNING);
+
+ int expectedEventsAtRunning = 3;
+ verify(eventHandler, times(expectedEventsAtRunning)).handle(arg.capture());
+
+ taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE));
+
+ assertEquals("Task attempt is not in the SUCCEEDED state", taImpl.getState(),
+ TaskAttemptState.SUCCEEDED);
+
+ assertEquals(0, taImpl.getDiagnostics().size());
+
+ int expectedEvenstAfterTerminating = expectedEventsAtRunning + 3;
+ arg = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture());
+
+ verifyEventType(
+ arg.getAllValues().subList(expectedEventsAtRunning,
+ expectedEvenstAfterTerminating), TaskEventTAUpdate.class, 1);
+ verifyEventType(
+ arg.getAllValues().subList(expectedEventsAtRunning,
+ expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1);
+
+ taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID,
+ "Terminated"));
+ int expectedEventAfterTerminated = expectedEvenstAfterTerminating + 0;
+ arg = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(expectedEventAfterTerminated)).handle(arg.capture());
+
+ // Verify that the diagnostic message included in the Terminated event is not
+ // captured - TA already succeeded.
+ assertEquals(0, taImpl.getDiagnostics().size());
+ }
+
+
+ @Test
// Verifies that multiple TooManyFetchFailures are handled correctly by the
// TaskAttempt.
public void testMultipleTooManyFetchFailures() throws Exception {
@@ -467,7 +675,8 @@ public class TestTaskAttempt {
TezTaskID taskID = new TezTaskID(vertexID, 1);
TezTaskAttemptID taskAttemptID = new TezTaskAttemptID(taskID, 0);
- MockEventHandler eventHandler = new MockEventHandler();
+ MockEventHandler mockEh = new MockEventHandler();
+ MockEventHandler eventHandler = spy(mockEh);
TaskAttemptListener taListener = mock(TaskAttemptListener.class);
when(taListener.getAddress()).thenReturn(
new InetSocketAddress("localhost", 0));
@@ -513,8 +722,20 @@ public class TestTaskAttempt {
TaskAttemptEventType.TA_DONE));
assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
TaskAttemptState.SUCCEEDED);
+
+ int expectedEventsTillSucceeded = 6;
+ ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(expectedEventsTillSucceeded)).handle(arg.capture());
+ verifyEventType(arg.getAllValues(), TaskEventTAUpdate.class, 2);
+
taImpl.handle(new TaskAttemptEvent(taskAttemptID,
TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES));
+ int expectedEventsAfterFetchFailure = expectedEventsTillSucceeded + 3;
+ verify(eventHandler, times(expectedEventsAfterFetchFailure)).handle(arg.capture());
+ verifyEventType(
+ arg.getAllValues().subList(expectedEventsTillSucceeded,
+ expectedEventsAfterFetchFailure), TaskEventTAUpdate.class, 1);
+
assertEquals("Task attempt is not in FAILED state", taImpl.getState(),
TaskAttemptState.FAILED);
taImpl.handle(new TaskAttemptEvent(taskAttemptID,
@@ -524,8 +745,24 @@ public class TestTaskAttempt {
assertFalse(
"InternalError occurred trying to handle TA_TOO_MANY_FETCH_FAILURES",
eventHandler.internalError);
+ // No new events.
+ verify(eventHandler, times(expectedEventsAfterFetchFailure)).handle(
+ arg.capture());
}
+ private void verifyEventType(List<Event> events,
+ Class<? extends Event> eventClass, int expectedOccurences) {
+ int count = 0;
+ for (Event e : events) {
+ if (eventClass.isInstance(e)) {
+ count++;
+ }
+ }
+ assertEquals(
+ "Mismatch in num occurences of event: " + eventClass.getCanonicalName(),
+ expectedOccurences, count);
+ }
+
public static class MockEventHandler implements EventHandler {
public boolean internalError;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d660948d/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 516ea5e..100f2aa 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -363,6 +363,8 @@ public class TestTaskImpl {
assertTaskSucceededState();
}
+
+ // TODO Add test to validate the correct commit attempt.
@SuppressWarnings("rawtypes")
private class MockTaskImpl extends TaskImpl {
[04/50] [abbrv] git commit: TEZ-25. cleanup of unused import
statements
Posted by ss...@apache.org.
TEZ-25. cleanup of unused import statements
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/45db1be3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/45db1be3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/45db1be3
Branch: refs/heads/master
Commit: 45db1be34f349dd7a2c94854d68d92a7e93a5719
Parents: cb5758b
Author: mikelid <mi...@microsoft.com>
Authored: Mon May 13 12:01:09 2013 -0700
Committer: mikelid <mi...@microsoft.com>
Committed: Mon May 13 12:01:09 2013 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/tez/dag/api/DAG.java | 5 -----
.../org/apache/tez/dag/api/DagTypeConverters.java | 4 ----
2 files changed, 0 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/45db1be3/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
index c18c720..efaf276 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -26,13 +26,8 @@ import java.util.Map.Entry;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.tez.dag.api.*;
-import org.apache.tez.dag.api.DAGPlan.PlanKeyValuePair.Builder;
import org.apache.tez.dag.api.DAGPlan.PlanTaskLocationHint;
-import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
-import org.apache.tez.dag.api.EdgeProperty.SourceType;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
import org.apache.tez.dag.api.DAGPlan.*;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/45db1be3/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index 04404ba..afbabad 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -17,15 +17,11 @@
*/
package org.apache.tez.dag.api;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
[28/50] [abbrv] git commit: TEZ-156. Fix build break due to removal
of BuilderUtils
Posted by ss...@apache.org.
TEZ-156. Fix build break due to removal of BuilderUtils
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/b5ebd492
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/b5ebd492
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/b5ebd492
Branch: refs/heads/master
Commit: b5ebd4923729dd3f0eb77339f2f1338265153588
Parents: 5d8adeb
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri May 24 10:13:42 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri May 24 10:13:42 2013 -0700
----------------------------------------------------------------------
.../dag/app/rm/container/AMContainerHelpers.java | 14 ++++++--------
.../org/apache/tez/mapreduce/NotRunningJob.java | 6 +++---
.../java/org/apache/tez/mapreduce/YARNRunner.java | 7 +++----
3 files changed, 12 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b5ebd492/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 0550895..6a9ae2e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -136,10 +136,9 @@ public class AMContainerHelpers {
// Construct the actual Container
// The null fields are per-container and will be constructed for each
// container separately.
- ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext(
- appContext.getDAG().getUserName(), localResources,
- environment, null, serviceData, taskCredentialsBuffer, applicationACLs);
-
+ ContainerLaunchContext container =
+ ContainerLaunchContext.newInstance(localResources, environment, null,
+ serviceData, taskCredentialsBuffer, applicationACLs);
return container;
}
@@ -188,10 +187,9 @@ public class AMContainerHelpers {
}
// Construct the actual Container
- ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext(
- commonContainerSpec.getUser(), lResources, myEnv, commands,
- myServiceData, commonContainerSpec.getTokens().duplicate(),
- acls);
+ ContainerLaunchContext container =
+ ContainerLaunchContext.newInstance(lResources, myEnv, commands,
+ myServiceData, commonContainerSpec.getTokens(), acls);
return container;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b5ebd492/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
index 370c8f0..ba7ab5a 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
@@ -87,9 +87,9 @@ public class NotRunningJob implements MRClientProtocol {
// Setting AppState to NEW and finalStatus to UNDEFINED as they are never
// used for a non running job
- return BuilderUtils.newApplicationReport(unknownAppId, unknownAttemptId,
- "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
- "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f);
+ return ApplicationReport.newInstance(unknownAppId, unknownAttemptId, "N/A",
+ "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", "N/A",
+ 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, "TEZ_MRR");
}
NotRunningJob(ApplicationReport applicationReport, JobState jobState) {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/b5ebd492/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 9ece649..a79a4f9 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -816,10 +816,9 @@ public class YARNRunner implements ClientProtocol {
= new HashMap<ApplicationAccessType, String>();
// Setup ContainerLaunchContext for AM container
- ContainerLaunchContext amContainer = BuilderUtils
- .newContainerLaunchContext(UserGroupInformation
- .getCurrentUser().getShortUserName(), localResources,
- environment, vargsFinal, null, securityTokens, acls);
+ ContainerLaunchContext amContainer =
+ ContainerLaunchContext.newInstance(localResources, environment,
+ vargsFinal, null, securityTokens, acls);
// Set up the ApplicationSubmissionContext
ApplicationSubmissionContext appContext = Records
[16/50] [abbrv] git commit: TEZ-144. Fix ContainerLaunch to always
use ContainerTokens, irrespective of security (YARN-617). (sseth)
Posted by ss...@apache.org.
TEZ-144. Fix ContainerLaunch to always use ContainerTokens, irrespective
of security (YARN-617). (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/0d5027d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/0d5027d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/0d5027d0
Branch: refs/heads/master
Commit: 0d5027d00d72292ce8acd45441d02b7d5e6e8ecb
Parents: de5ee03
Author: Siddharth Seth <ss...@apache.org>
Authored: Sat May 18 20:43:36 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Sat May 18 20:43:36 2013 -0700
----------------------------------------------------------------------
.../dag/app/launcher/ContainerLauncherImpl.java | 16 ++++++--------
1 files changed, 7 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/0d5027d0/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index 48a41c4..f1a16b8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -360,15 +360,13 @@ public class ContainerLauncherImpl extends AbstractService implements
final InetSocketAddress cmAddr =
NetUtils.createSocketAddr(containerManagerBindAddr);
- UserGroupInformation user = UserGroupInformation.getCurrentUser();
-
- if (UserGroupInformation.isSecurityEnabled()) {
- Token<ContainerTokenIdentifier> token =
- ProtoUtils.convertFromProtoFormat(containerToken, cmAddr);
- // the user in createRemoteUser in this context has to be ContainerID
- user = UserGroupInformation.createRemoteUser(containerID.toString());
- user.addToken(token);
- }
+
+ // the user in createRemoteUser in this context has to be ContainerID
+ UserGroupInformation user = UserGroupInformation
+ .createRemoteUser(containerID.toString());
+ Token<ContainerTokenIdentifier> token = ProtoUtils.convertFromProtoFormat(
+ containerToken, cmAddr);
+ user.addToken(token);
ContainerManager proxy = user
.doAs(new PrivilegedAction<ContainerManager>() {
[25/50] [abbrv] git commit: TEZ-32. Fix logging support for MapReduce
tasks and AM. (sseth)
Posted by ss...@apache.org.
TEZ-32. Fix logging support for MapReduce tasks and AM. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/268f3ac9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/268f3ac9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/268f3ac9
Branch: refs/heads/master
Commit: 268f3ac97b081294fcdc22f7258890f219285fca
Parents: 36198a6
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu May 23 11:17:43 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu May 23 11:17:43 2013 -0700
----------------------------------------------------------------------
.../org/apache/tez/dag/api/TezConfiguration.java | 4 +-
.../apache/tez/dag/utils/TezEngineChildJVM.java | 1 -
.../java/org/apache/tez/mapreduce/YARNRunner.java | 113 ++++++++-------
3 files changed, 61 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/268f3ac9/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 4a407b3..3018ea1 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -45,7 +45,9 @@ public class TezConfiguration extends Configuration {
// TODO Should not be required once all tokens are handled via AppSubmissionContext
public static final String JOB_SUBMIT_DIR = TEZ_PREFIX + "jobSubmitDir";
public static final String APPLICATION_TOKENS_FILE = "appTokens";
-
+ public static final String DAG_APPLICATION_MASTER_CLASS =
+ "org.apache.tez.dag.app.DAGAppMaster";
+
public static final String DAG_AM_TASK_LISTENER_THREAD_COUNT =
TEZ_PREFIX + "task.listener.thread-count";
public static final int DAG_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT = 30;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/268f3ac9/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
index 727ded5..bc92ed8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.app.rm.container.AMContainerHelpers;
import org.apache.tez.dag.records.TezVertexID;
public class TezEngineChildJVM {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/268f3ac9/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 7582a5e..9ece649 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -45,8 +44,6 @@ import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TaskLog;
-import org.apache.hadoop.mapred.TaskLog.LogName;
import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.mapreduce.ClusterMetrics;
import org.apache.hadoop.mapreduce.Counters;
@@ -70,7 +67,6 @@ import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIden
import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader;
import org.apache.hadoop.mapreduce.v2.LogParams;
-import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.Credentials;
@@ -93,11 +89,10 @@ import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.Edge;
@@ -120,8 +115,6 @@ public class YARNRunner implements ClientProtocol {
private static final Log LOG = LogFactory.getLog(YARNRunner.class);
- private final RecordFactory recordFactory =
- RecordFactoryProvider.getRecordFactory(null);
private ResourceMgrDelegate resMgrDelegate;
private ClientCache clientCache;
private Configuration conf;
@@ -172,6 +165,7 @@ public class YARNRunner implements ClientProtocol {
}
}
+ @VisibleForTesting
@Private
/**
* Used for testing mostly.
@@ -210,12 +204,6 @@ public class YARNRunner implements ClientProtocol {
return resMgrDelegate.getClusterMetrics();
}
- @VisibleForTesting
- Token<?> getDelegationTokenFromHS(MRClientProtocol hsProxy)
- throws IOException, InterruptedException {
- throw new UnsupportedOperationException("No HistoryServer for Tez");
- }
-
@Override
public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
throws IOException, InterruptedException {
@@ -384,17 +372,6 @@ public class YARNRunner implements ClientProtocol {
MRJobConfig.MAPRED_ADMIN_USER_ENV,
MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV));
- // FIXME is this really required?
- // Add stdout/stderr env
- environment.put(
- MRJobConfig.STDOUT_LOGFILE_ENV,
- getTaskLogFile(TaskLog.LogName.STDOUT)
- );
- environment.put(
- MRJobConfig.STDERR_LOGFILE_ENV,
- getTaskLogFile(TaskLog.LogName.STDERR)
- );
-
}
private static String getChildEnv(Configuration jobConf, boolean isMap) {
@@ -418,19 +395,7 @@ public class YARNRunner implements ClientProtocol {
}
}
- private static String getTaskLogFile(LogName filter) {
- return ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR
- + filter.toString();
- }
-
- private static void setupLog4jProperties(Configuration jobConf,
- boolean isMap,
- Vector<String> vargs,
- long logSize) {
- String logLevel = getChildLogLevel(jobConf, isMap);
- MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
- }
-
+
private void setupMapReduceEnv(Configuration jobConf,
Map<String, String> environment, boolean isMap) throws IOException {
@@ -474,13 +439,10 @@ public class YARNRunner implements ClientProtocol {
// FIXME: don't think this is also needed given we already set java
// properties.
// TODO Change this not to use JobConf.
- long logSize = TaskLog.getTaskLogLength(new JobConf(jobConf));
- Vector<String> logProps = new Vector<String>(4);
- setupLog4jProperties(jobConf, isMap, logProps, logSize);
- Iterator<String> it = logProps.iterator();
+ String log4jCmdLineProperties = getLog4jCmdLineProperties(jobConf, isMap);
StringBuffer buffer = new StringBuffer();
- while (it.hasNext()) {
- buffer.append(" " + it.next());
+ if (log4jCmdLineProperties != null && log4jCmdLineProperties != "") {
+ buffer.append(" " + log4jCmdLineProperties);
}
// FIXME supposedly required for streaming, should we remove it and let
@@ -492,7 +454,7 @@ public class YARNRunner implements ClientProtocol {
hadoopClientOpts = hadoopClientOpts + " ";
}
hadoopClientOpts = hadoopClientOpts + buffer.toString();
- // environment.put("HADOOP_CLIENT_OPTS", hadoopClientOpts);
+ //environment.put("HADOOP_CLIENT_OPTS", hadoopClientOpts);
// FIXME for this to work, we need YARN-561 and the task runtime changed
// to use YARN-561
@@ -730,7 +692,7 @@ public class YARNRunner implements ClientProtocol {
ApplicationId applicationId = resMgrDelegate.getApplicationId();
// Setup resource requirements
- Resource capability = recordFactory.newRecordInstance(Resource.class);
+ Resource capability = Records.newRecord(Resource.class);
capability.setMemory(
conf.getInt(TezConfiguration.DAG_AM_RESOURCE_MEMORY_MB,
TezConfiguration.DEFAULT_DAG_AM_RESOURCE_MEMORY_MB));
@@ -757,9 +719,10 @@ public class YARNRunner implements ClientProtocol {
// LOG.error(" !!!!!!!!!");
// vargs.add("-Xdebug -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=8002,server=y,suspend=y");
- // FIXME set up logging related properties
// TODO -Dtez.root.logger??
- // MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
+ String amLogLevel = jobConf.get(MRJobConfig.MR_AM_LOG_LEVEL,
+ MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
+ addLog4jSystemProperties(amLogLevel, vargs);
// FIXME admin command opts and user command opts for tez?
String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
@@ -775,7 +738,7 @@ public class YARNRunner implements ClientProtocol {
MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
vargs.add(mrAppMasterUserOptions);
- vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS);
+ vargs.add(TezConfiguration.DAG_APPLICATION_MASTER_CLASS);
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
Path.SEPARATOR + ApplicationConstants.STDOUT);
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
@@ -859,8 +822,9 @@ public class YARNRunner implements ClientProtocol {
environment, vargsFinal, null, securityTokens, acls);
// Set up the ApplicationSubmissionContext
- ApplicationSubmissionContext appContext =
- recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
+ ApplicationSubmissionContext appContext = Records
+ .newRecord(ApplicationSubmissionContext.class);
+
appContext.setApplicationId(applicationId); // ApplicationId
appContext.setResource(capability); // resource
appContext.setQueue( // Queue name
@@ -975,7 +939,7 @@ public class YARNRunner implements ClientProtocol {
private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type)
throws IOException {
- LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
+ LocalResource rsrc = Records.newRecord(LocalResource.class);
FileStatus rsrcStat = fs.getFileStatus(p);
rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
.getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
@@ -1127,6 +1091,7 @@ public class YARNRunner implements ClientProtocol {
}
}
+ @SuppressWarnings("deprecation")
private String getMapJavaOpts(Configuration jobConf) {
// follows pattern from YARN MapReduceChildJVM.java
String adminOpts = "";
@@ -1142,9 +1107,11 @@ public class YARNRunner implements ClientProtocol {
JobConf.MAPRED_TASK_JAVA_OPTS,
JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
- return adminOpts + " " + userOpts;
+ return adminOpts.trim() + " " + userOpts.trim() + " "
+ + getLog4jCmdLineProperties(jobConf, true);
}
+ @SuppressWarnings("deprecation")
private String getReduceJavaOpts(Configuration jobConf) {
// follows pattern from YARN MapReduceChildJVM.java
String adminOpts = "";
@@ -1159,6 +1126,42 @@ public class YARNRunner implements ClientProtocol {
jobConf.get(
JobConf.MAPRED_TASK_JAVA_OPTS,
JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
- return adminOpts + " " + userOpts;
+
+ return adminOpts.trim() + " " + userOpts.trim() + " "
+ + getLog4jCmdLineProperties(jobConf, false);
}
-}
+
+ private static String getLog4jCmdLineProperties(Configuration jobConf,
+ boolean isMap) {
+ Vector<String> logProps = new Vector<String>(4);
+ addLog4jSystemProperties(getChildLogLevel(jobConf, isMap), logProps);
+ StringBuilder sb = new StringBuilder();
+ for (String str : logProps) {
+ sb.append(str).append(" ");
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Add the JVM system properties necessary to configure
+ * {@link ContainerLogAppender}.
+ *
+ * @param logLevel
+ * the desired log level (eg INFO/WARN/DEBUG)
+ * @param vargs
+ * the argument list to append to
+ */
+ private static void addLog4jSystemProperties(String logLevel,
+ List<String> vargs) {
+ vargs.add("-Dlog4j.configuration=container-log4j.properties");
+ // TODO Fix the remaining properties after YARN-720 is fixed.
+ // May need to introduce a log4j properties file for tez tasks.
+ vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "="
+ + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ // Setting this to 0 to avoid log size restrictions.
+ // Should be enforced by YARN.
+ vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + 0);
+ vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
+ }
+
+}
\ No newline at end of file
[19/50] [abbrv] git commit: TEZ-149. MRRuntimeTask should read local
dir configuration from config rather than from env. (sseth)
Posted by ss...@apache.org.
TEZ-149. MRRuntimeTask should read local dir configuration from config
rather than from env. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/9fb21703
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/9fb21703
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/9fb21703
Branch: refs/heads/master
Commit: 9fb21703c8bd73a5ffd15a39d7a598bcc0ce32c7
Parents: 8298190
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue May 21 11:28:06 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue May 21 11:28:06 2013 -0700
----------------------------------------------------------------------
.../common/sort/impl/dflt/DefaultSorter.java | 2 +-
.../processor/reduce/ReduceProcessor.java | 2 +-
.../apache/tez/mapreduce/task/MRRuntimeTask.java | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fb21703/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
index 2cf5ab4..e567492 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
@@ -219,7 +219,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
}
if (partition < 0 || partition >= partitions) {
throw new IOException("Illegal partition for " + key + " (" +
- partition + ")");
+ partition + ")" + ", TotalPartitions: " + partitions);
}
checkSpillException();
bufferRemaining -= METASIZE;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fb21703/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
index 9ae07ee..ce49726 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
@@ -303,7 +303,7 @@ implements Processor {
org.apache.hadoop.mapreduce.Reducer reducer =
(org.apache.hadoop.mapreduce.Reducer)
ReflectionUtils.newInstance(taskContext.getReducerClass(), job);
-
+
org.apache.hadoop.mapreduce.RecordWriter trackedRW =
new org.apache.hadoop.mapreduce.RecordWriter() {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9fb21703/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index 6091d4b..3b51889 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -198,7 +198,7 @@ public class MRRuntimeTask extends RuntimeTask {
private static void setupDistributedCacheConfig(final JobConf job)
throws IOException {
- String localWorkDir = System.getenv(System.getenv(Environment.PWD.name()));
+ String localWorkDir = (job.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
// ^ ^ all symlinks are created in the current work-dir
// Update the configuration object with localized archives.
[09/50] [abbrv] git commit: TEZ-127. YARN-615 build break fix (bikas)
Posted by ss...@apache.org.
TEZ-127. YARN-615 build break fix (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/4c89e15c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/4c89e15c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/4c89e15c
Branch: refs/heads/master
Commit: 4c89e15c61787742326ee943ba17ab61b65427fc
Parents: 8193b5f
Author: Bikas Saha <bi...@apache.org>
Authored: Tue May 14 14:34:15 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue May 14 14:34:15 2013 -0700
----------------------------------------------------------------------
.../dag/app/rm/container/AMContainerHelpers.java | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c89e15c/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 5eca572..0550895 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -190,7 +190,7 @@ public class AMContainerHelpers {
// Construct the actual Container
ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext(
commonContainerSpec.getUser(), lResources, myEnv, commands,
- myServiceData, commonContainerSpec.getContainerTokens().duplicate(),
+ myServiceData, commonContainerSpec.getTokens().duplicate(),
acls);
return container;
[38/50] [abbrv] git commit: TEZ-171: MapUtils should use the temp-dir
passed in rather than env:PWD
Posted by ss...@apache.org.
TEZ-171: MapUtils should use the temp-dir passed in rather than env:PWD
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/5d0b8adf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/5d0b8adf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/5d0b8adf
Branch: refs/heads/master
Commit: 5d0b8adfb9390b67c53e9bbad7a9f0961300d8db
Parents: 6d5625f
Author: Mike Liddell <ml...@apache.org>
Authored: Thu May 30 16:45:30 2013 -0700
Committer: Mike Liddell <ml...@apache.org>
Committed: Thu May 30 16:45:30 2013 -0700
----------------------------------------------------------------------
.../apache/tez/mapreduce/processor/MapUtils.java | 3 +--
1 files changed, 1 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5d0b8adf/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
index 48a1113..138a8e0 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
@@ -66,8 +66,7 @@ public class MapUtils {
localSysDirs[0] = localDir;
conf.setStrings(TezJobConfig.LOCAL_DIRS, localSysDirs);
- conf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
- System.getenv(Environment.PWD.name()));
+ conf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, localDir);
LOG.info(TezJobConfig.LOCAL_DIRS + " for child: "
+ conf.get(TezJobConfig.LOCAL_DIRS));
[50/50] [abbrv] git commit: Merge branch 'master' into TEZ-1
Posted by ss...@apache.org.
Merge branch 'master' into TEZ-1
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/809bbd96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/809bbd96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/809bbd96
Branch: refs/heads/master
Commit: 809bbd9643427f7558669f6443b7b0e879191b8b
Parents: a10e607 d6cd24b
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jun 3 22:26:46 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jun 3 22:26:46 2013 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[22/50] [abbrv] git commit: TEZ-9. Support querying state of the DAG
AM from a client via a client RPC (bikas)
Posted by ss...@apache.org.
TEZ-9. Support querying state of the DAG AM from a client via a client RPC (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/ef37823c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/ef37823c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/ef37823c
Branch: refs/heads/master
Commit: ef37823c90efedb5a0b14f94b4e9e785cb1c5071
Parents: 37d7a4c
Author: Bikas Saha <bi...@apache.org>
Authored: Tue May 21 18:28:41 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue May 21 18:28:41 2013 -0700
----------------------------------------------------------------------
tez-dag-api/pom.xml | 15 ++
.../main/java/org/apache/tez/client/TezClient.java | 95 +++++++++++++
.../org/apache/tez/dag/api/TezConfiguration.java | 11 ++
.../java/org/apache/tez/dag/api/TezException.java | 2 +-
.../org/apache/tez/dag/api/TezRemoteException.java | 31 +++++
.../org/apache/tez/dag/api/client/DAGClient.java | 10 +-
.../org/apache/tez/dag/api/client/Progress.java | 9 ++
.../tez/dag/api/client/rpc/DAGClientRPCImpl.java | 18 ++-
.../apache/tez/dag/api/client/DAGClientServer.java | 106 +++++++++++++++
.../DAGClientAMProtocolBlockingPBServerImpl.java | 103 ++++++++++++++
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 49 ++++---
.../apache/tez/dag/app/client/ClientService.java | 29 ----
.../tez/dag/app/client/impl/TezClientService.java | 43 ------
.../apache/tez/dag/app/client/package-info.java | 20 ---
.../tez/dag/app/rm/TaskSchedulerEventHandler.java | 31 +++--
.../tez/mapreduce/ClientServiceDelegate.java | 70 ++++++++--
.../org/apache/tez/mapreduce/DAGJobStatus.java | 27 ++++-
17 files changed, 517 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag-api/pom.xml b/tez-dag-api/pom.xml
index 74b1284..3cb0952 100644
--- a/tez-dag-api/pom.xml
+++ b/tez-dag-api/pom.xml
@@ -34,6 +34,10 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</dependency>
</dependencies>
@@ -46,6 +50,17 @@
<configuration>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.tez.client.TezClient</mainClass>
+ </manifest>
+ </archive>
+ </configuration>
+ </plugin>
<plugin>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
new file mode 100644
index 0000000..f794014
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -0,0 +1,95 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.client;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.client.YarnClient;
+import org.apache.hadoop.yarn.client.YarnClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezRemoteException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.client.rpc.DAGClientRPCImpl;
+
+public class TezClient {
+ private final TezConfiguration conf;
+
+ public TezClient(TezConfiguration conf) {
+ this.conf = conf;
+ }
+
+ public DAGClient getDAGClient(String appIdStr) throws IOException, TezException {
+ try {
+ System.out.println("Fetching app: " + appIdStr);
+ ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
+ YarnClient yarnClient = new YarnClientImpl();
+ yarnClient.init(conf);
+ yarnClient.start();
+ ApplicationReport appReport = yarnClient.getApplicationReport(appId);
+ String host = appReport.getHost();
+ int port = appReport.getRpcPort();
+ return getDAGClient(host, port);
+ } catch (YarnRemoteException e) {
+ throw new TezException(e);
+ }
+ }
+
+ public DAGClient getDAGClient(String host, int port) throws IOException {
+ System.out.println("App host port: " + host + ":" + port);
+ InetSocketAddress addr = new InetSocketAddress(host, port);
+ DAGClient dagClient;
+ dagClient = new DAGClientRPCImpl(1, addr, conf);
+ return dagClient;
+ }
+
+ public static void main(String[] args) {
+ try {
+ TezClient tezClient = new TezClient(
+ new TezConfiguration(new YarnConfiguration()));
+ DAGClient dagClient = tezClient.getDAGClient(args[1]);
+ String dagId = dagClient.getAllDAGs().get(0);
+ DAGStatus dagStatus = dagClient.getDAGStatus(dagId);
+ System.out.println("DAG: " + dagId +
+ " State: " + dagStatus.getState() +
+ " Progress: " + dagStatus.getDAGProgress());
+ for(String vertexName : dagStatus.getVertexProgress().keySet()) {
+ System.out.println("VertexStatus from DagStatus:" +
+ " Vertex: " + vertexName +
+ " Progress: " + dagStatus.getVertexProgress().get(vertexName));
+ VertexStatus vertexStatus = dagClient.getVertexStatus(dagId, vertexName);
+ System.out.println("VertexStatus:" +
+ " Vertex: " + vertexName +
+ " State: " + vertexStatus.getState() +
+ " Progress: " + vertexStatus.getProgress());
+ }
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 597fb2f..4a407b3 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -65,6 +65,17 @@ public class TezConfiguration extends Configuration {
public static final String DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD = TEZ_PREFIX
+ "node-blacklisting.ignore-threshold-node-percent";
public static final int DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT = 33;
+
+ /** Number of threads to handle job client RPC requests.*/
+ public static final String DAG_CLIENT_AM_THREAD_COUNT =
+ TEZ_PREFIX + "client.am.thread-count";
+ public static final int DAG_CLIENT_AM__THREAD_COUNT_DEFAULT = 1;
+ /**
+ * Range of ports that the AM can use when binding. Leave blank
+ * if you want all possible ports.
+ */
+ public static final String DAG_CLIENT_AM_PORT_RANGE =
+ TEZ_PREFIX + "client.am.port-range";
public static final String DAG_AM_RESOURCE_MEMORY_MB = DAG_AM_PREFIX
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java
index 265b89c..3014309 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezException.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.api;
-/*
+/**
* Base Tez Exception
*/
public class TezException extends RuntimeException {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezRemoteException.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezRemoteException.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezRemoteException.java
new file mode 100644
index 0000000..8311852
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezRemoteException.java
@@ -0,0 +1,31 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api;
+
+/**
+ * Base TezRemoteException
+ */
+public class TezRemoteException extends Exception {
+ private static final long serialVersionUID = 6337442733802964447L;
+ public TezRemoteException(Throwable cause) { super(cause); }
+ public TezRemoteException(String message) { super(message); }
+ public TezRemoteException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
index 477b017..0e9e55b 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
@@ -18,9 +18,10 @@
package org.apache.tez.dag.api.client;
+import java.io.IOException;
import java.util.List;
-import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezRemoteException;
/*
* Inteface class for monitoring the <code>DAG</code> running in a Tez DAG
@@ -31,16 +32,17 @@ public interface DAGClient {
/**
* Return the identifiers for all DAG's
*/
- List<String> getAllDAGs() throws TezException;
+ List<String> getAllDAGs() throws IOException, TezRemoteException;
/**
* Get the status of a DAG
*/
- DAGStatus getDAGStatus(String dagId) throws TezException;
+ DAGStatus getDAGStatus(String dagId) throws IOException, TezRemoteException;
/**
* Get the status of a Vertex of a DAG
*/
VertexStatus getVertexStatus(String dagId,
- String vertexName) throws TezException;
+ String vertexName)
+ throws IOException, TezRemoteException;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
index 7158e30..af9f2a4 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
@@ -47,5 +47,14 @@ public class Progress {
public int getKilledTaskCount() {
return proxy.getKilledTaskCount();
}
+
+ @Override
+ public String toString() {
+ return new String("Total: " + getTotalTaskCount() +
+ " Succeeded: " + getSucceededTaskCount() +
+ " Running: " + getRunningTaskCount() +
+ " Failed: " + getFailedTaskCount() +
+ " Killed: " + getKilledTaskCount());
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
index 9238f63..42a815e 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
@@ -26,7 +26,7 @@ import java.util.List;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezRemoteException;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.VertexStatus;
@@ -55,18 +55,20 @@ public class DAGClientRPCImpl implements DAGClient, Closeable {
}
@Override
- public List<String> getAllDAGs() throws TezException {
+ public List<String> getAllDAGs() throws IOException, TezRemoteException {
GetAllDAGsRequestProto requestProto =
GetAllDAGsRequestProto.newBuilder().build();
try {
return proxy.getAllDAGs(null, requestProto).getDagIdList();
} catch (ServiceException e) {
- throw new TezException(e);
+ // TEZ-151 retrieve wrapped TezRemoteException
+ throw new TezRemoteException(e);
}
}
@Override
- public DAGStatus getDAGStatus(String dagId) throws TezException {
+ public DAGStatus getDAGStatus(String dagId)
+ throws IOException, TezRemoteException {
GetDAGStatusRequestProto requestProto =
GetDAGStatusRequestProto.newBuilder().setDagId(dagId).build();
@@ -74,13 +76,14 @@ public class DAGClientRPCImpl implements DAGClient, Closeable {
return new DAGStatus(
proxy.getDAGStatus(null, requestProto).getDagStatus());
} catch (ServiceException e) {
- throw new TezException(e);
+ // TEZ-151 retrieve wrapped TezRemoteException
+ throw new TezRemoteException(e);
}
}
@Override
public VertexStatus getVertexStatus(String dagId, String vertexName)
- throws TezException {
+ throws IOException, TezRemoteException {
GetVertexStatusRequestProto requestProto =
GetVertexStatusRequestProto.newBuilder().
setDagId(dagId).setVertexName(vertexName).build();
@@ -89,7 +92,8 @@ public class DAGClientRPCImpl implements DAGClient, Closeable {
return new VertexStatus(
proxy.getVertexStatus(null, requestProto).getVertexStatus());
} catch (ServiceException e) {
- throw new TezException(e);
+ // TEZ-151 retrieve wrapped TezRemoteException
+ throw new TezRemoteException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java
new file mode 100644
index 0000000..534fcaa
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java
@@ -0,0 +1,106 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPBServerImpl;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.DAGClientAMProtocol;
+
+import com.google.protobuf.BlockingService;
+
+public class DAGClientServer extends AbstractService {
+ static final Log LOG = LogFactory.getLog(DAGClientServer.class);
+
+ DAGClient realInstance;
+ Server server;
+ InetSocketAddress bindAddress;
+
+ public DAGClientServer(DAGClient realInstance) {
+ super("DAGClientRPCServer");
+ this.realInstance = realInstance;
+ }
+
+ @Override
+ public void start() {
+ try {
+ assert getConfig() instanceof TezConfiguration;
+ TezConfiguration conf = (TezConfiguration) getConfig();
+ InetSocketAddress addr = new InetSocketAddress(0);
+
+ DAGClientAMProtocolBlockingPBServerImpl service =
+ new DAGClientAMProtocolBlockingPBServerImpl(realInstance);
+
+ BlockingService blockingService =
+ DAGClientAMProtocol.newReflectiveBlockingService(service);
+
+ int numHandlers = conf.getInt(TezConfiguration.DAG_CLIENT_AM_THREAD_COUNT,
+ TezConfiguration.DAG_CLIENT_AM__THREAD_COUNT_DEFAULT);
+
+ String portRange = conf.get(TezConfiguration.DAG_CLIENT_AM_PORT_RANGE);
+
+ server = createServer(DAGClientAMProtocolBlockingPB.class, addr, conf,
+ numHandlers, blockingService, portRange);
+ server.start();
+ bindAddress = NetUtils.getConnectAddress(server);
+ LOG.info("Instantiated DAGClientRPCServer at " + bindAddress);
+ super.start();
+ } catch (Exception e) {
+ LOG.error("Failed to start DAGClientServer: ", e);
+ throw new TezException(e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ if(server != null) {
+ server.stop();
+ }
+ super.stop();
+ }
+
+ public InetSocketAddress getBindAddress() {
+ return bindAddress;
+ }
+
+ private Server createServer(Class<?> pbProtocol, InetSocketAddress addr, Configuration conf,
+ int numHandlers,
+ BlockingService blockingService, String portRangeConfig) throws IOException {
+ RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class);
+ RPC.Server server = new RPC.Builder(conf).setProtocol(pbProtocol)
+ .setInstance(blockingService).setBindAddress(addr.getHostName())
+ .setPort(addr.getPort()).setNumHandlers(numHandlers).setVerbose(false)
+ .setPortRangeConfig(portRangeConfig)
+ .build();
+ server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, pbProtocol, blockingService);
+ return server;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java
new file mode 100644
index 0000000..81d7868
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java
@@ -0,0 +1,103 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client.rpc;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.tez.dag.api.TezRemoteException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.DAGStatusBuilder;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.client.VertexStatusBuilder;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAllDAGsRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAllDAGsResponseProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetDAGStatusRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetDAGStatusResponseProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetVertexStatusRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetVertexStatusResponseProto;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+
+public class DAGClientAMProtocolBlockingPBServerImpl implements
+ DAGClientAMProtocolBlockingPB {
+
+ DAGClient real;
+
+ public DAGClientAMProtocolBlockingPBServerImpl(DAGClient real) {
+ this.real = real;
+ }
+
+ @Override
+ public GetAllDAGsResponseProto getAllDAGs(RpcController controller,
+ GetAllDAGsRequestProto request) throws ServiceException {
+ try{
+ List<String> dagIds = real.getAllDAGs();
+ return GetAllDAGsResponseProto.newBuilder().addAllDagId(dagIds).build();
+ } catch(TezRemoteException e) {
+ throw wrapException(e);
+ } catch(IOException e) {
+ throw wrapException(e);
+ }
+ }
+
+ @Override
+ public GetDAGStatusResponseProto getDAGStatus(RpcController controller,
+ GetDAGStatusRequestProto request) throws ServiceException {
+ try {
+ String dagId = request.getDagId();
+ DAGStatus status;
+ status = real.getDAGStatus(dagId);
+ assert status instanceof DAGStatusBuilder;
+ DAGStatusBuilder builder = (DAGStatusBuilder) status;
+ return GetDAGStatusResponseProto.newBuilder().
+ setDagStatus(builder.getProto()).build();
+ } catch (TezRemoteException e) {
+ throw wrapException(e);
+ } catch(IOException e) {
+ throw wrapException(e);
+ }
+ }
+
+ @Override
+ public GetVertexStatusResponseProto getVertexStatus(RpcController controller,
+ GetVertexStatusRequestProto request) throws ServiceException {
+ try {
+ String dagId = request.getDagId();
+ String vertexName = request.getVertexName();
+ VertexStatus status = real.getVertexStatus(dagId, vertexName);
+ assert status instanceof VertexStatusBuilder;
+ VertexStatusBuilder builder = (VertexStatusBuilder) status;
+ return GetVertexStatusResponseProto.newBuilder().
+ setVertexStatus(builder.getProto()).build();
+ } catch (TezRemoteException e) {
+ throw wrapException(e);
+ } catch(IOException e) {
+ throw wrapException(e);
+ }
+ }
+
+ ServiceException wrapException(Exception e){
+ return new ServiceException(e);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 804c20c..821dd71 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -67,13 +67,12 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGClientServer;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.VertexStatus;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.app.client.ClientService;
-import org.apache.tez.dag.app.client.impl.TezClientService;
+import org.apache.tez.dag.api.TezRemoteException;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttempt;
@@ -157,7 +156,6 @@ public class DAGAppMaster extends CompositeService {
private AppContext context;
private TezConfiguration conf;
private Dispatcher dispatcher;
- private ClientService clientService;
// TODO Recovery
//private Recovery recoveryServ;
private ContainerLauncher containerLauncher;
@@ -178,7 +176,9 @@ public class DAGAppMaster extends CompositeService {
private HistoryEventHandler historyEventHandler;
private DAGAppMasterState state;
- private DAGMonitorServer monitor;
+
+ DAGClientServer clientRpcServer;
+ private DAGClientHandler clientHandler;
private DAG dag;
private Credentials fsTokens = new Credentials(); // Filled during init
@@ -230,13 +230,16 @@ public class DAGAppMaster extends CompositeService {
dagId = new TezDAGID(appAttemptID.getApplicationId(), 1);
- monitor = new DAGMonitorServer();
+ clientHandler = new DAGClientHandler();
// TODO Committer.
// committer = createOutputCommitter(conf);
dispatcher = createDispatcher();
addIfService(dispatcher);
+
+ clientRpcServer = new DAGClientServer(clientHandler);
+ addIfService(clientRpcServer);
taskHeartbeatHandler = createTaskHeartbeatHandler(context, conf);
addIfService(taskHeartbeatHandler);
@@ -262,11 +265,6 @@ public class DAGAppMaster extends CompositeService {
taskCleaner = createTaskCleaner(context);
addIfService(taskCleaner);
- // TODO TEZ-9
- //service to handle requests from JobClient
- clientService = new TezClientService();
- addIfService(clientService);
-
this.dagEventDispatcher = new DagEventDispatcher();
this.vertexEventDispatcher = new VertexEventDispatcher();
@@ -287,7 +285,7 @@ public class DAGAppMaster extends CompositeService {
dispatcher.register(NMCommunicatorEventType.class, containerLauncher);
taskSchedulerEventHandler = new TaskSchedulerEventHandler(context,
- clientService);
+ clientRpcServer);
addIfService(taskSchedulerEventHandler);
dispatcher.register(AMSchedulerEventType.class,
taskSchedulerEventHandler);
@@ -712,30 +710,37 @@ public class DAGAppMaster extends CompositeService {
LOG.info("On DAG completion. Old state: " + oldState + " new state: " + state);
}
- class DAGMonitorServer implements DAGClient {
+ class DAGClientHandler implements DAGClient {
@Override
- public List<String> getAllDAGs() {
+ public List<String> getAllDAGs() throws TezRemoteException {
return Collections.singletonList(dag.getID().toString());
}
@Override
- public DAGStatus getDAGStatus(String dagIdStr) throws TezException {
+ public DAGStatus getDAGStatus(String dagIdStr)
+ throws IOException, TezRemoteException {
return getDAG(dagIdStr).getDAGStatus();
}
@Override
- public VertexStatus getVertexStatus(String dagIdStr, String vertexName) {
- return getDAG(dagIdStr).getVertexStatus(vertexName);
+ public VertexStatus getVertexStatus(String dagIdStr, String vertexName)
+ throws IOException, TezRemoteException{
+ VertexStatus status = getDAG(dagIdStr).getVertexStatus(vertexName);
+ if(status == null) {
+ throw new TezRemoteException("Unknown vertexName: " + vertexName);
+ }
+
+ return status;
}
- DAG getDAG(String dagIdStr) {
+ DAG getDAG(String dagIdStr) throws IOException, TezRemoteException {
TezDAGID dagId = TezDAGID.fromString(dagIdStr);
if(dagId == null) {
- throw new TezException("Bad dagId: " + dagIdStr);
+ throw new TezRemoteException("Bad dagId: " + dagIdStr);
}
if(!dagId.equals(dag.getID())) {
- throw new TezException("Unknown dagId: " + dagIdStr);
+ throw new TezRemoteException("Unknown dagId: " + dagIdStr);
}
return dag;
}
@@ -1083,7 +1088,9 @@ public class DAGAppMaster extends CompositeService {
// that it doesnt take too long in shutting down
// Signal the task scheduler.
- appMaster.taskSchedulerEventHandler.setSignalled(true);
+ if(appMaster.getServiceState() == STATE.STARTED) {
+ appMaster.taskSchedulerEventHandler.setSignalled(true);
+ }
if (EnumSet.of(DAGAppMasterState.NEW, DAGAppMasterState.INITED,
DAGAppMasterState.RUNNING).contains(appMaster.state)) {
// DAG not in a final state. Must have receive a KILL signal
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/app/client/ClientService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/client/ClientService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/client/ClientService.java
deleted file mode 100644
index 64937a9..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/client/ClientService.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.client;
-
-import java.net.InetSocketAddress;
-
-// TODONOTES - RPC service for clients
-public interface ClientService {
-
- InetSocketAddress getBindAddress();
-
- int getHttpPort();
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/app/client/impl/TezClientService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/client/impl/TezClientService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/client/impl/TezClientService.java
deleted file mode 100644
index 8fd79f0..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/client/impl/TezClientService.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.client.impl;
-
-import java.net.InetSocketAddress;
-
-import org.apache.tez.dag.app.client.ClientService;
-
-public class TezClientService implements ClientService {
-
- // TODO remove dummy client service
- private final InetSocketAddress dummySocketAddress =
- new InetSocketAddress(0);
-
- @Override
- public InetSocketAddress getBindAddress() {
- // TODO Auto-generated method stub
- return dummySocketAddress;
- }
-
- @Override
- public int getHttpPort() {
- // TODO Auto-generated method stub
- return 0;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/app/client/package-info.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/client/package-info.java b/tez-dag/src/main/java/org/apache/tez/dag/app/client/package-info.java
deleted file mode 100644
index a4cd9cc..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/client/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-@InterfaceAudience.Private
-package org.apache.tez.dag.app.client;
-import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 85ba0b2..c61cbcc 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -38,11 +38,11 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.tez.dag.api.client.DAGClientServer;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.DAGAppMaster;
import org.apache.tez.dag.app.DAGAppMasterState;
-import org.apache.tez.dag.app.client.ClientService;
import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.app.dag.event.DAGAppMasterEvent;
import org.apache.tez.dag.app.dag.event.DAGAppMasterEventType;
@@ -69,31 +69,24 @@ public class TaskSchedulerEventHandler extends AbstractService
protected final AppContext appContext;
@SuppressWarnings("rawtypes")
private final EventHandler eventHandler;
- private final TaskScheduler taskScheduler;
- // TODO change this to DAGAppMaster
+ private TaskScheduler taskScheduler;
private DAGAppMaster dagAppMaster;
private Map<ApplicationAccessType, String> appAcls = null;
private Thread eventHandlingThread;
private volatile boolean stopEventHandling;
// Has a signal (SIGTERM etc) been issued?
protected volatile boolean isSignalled = false;
+ final DAGClientServer clientService;
BlockingQueue<AMSchedulerEvent> eventQueue
= new LinkedBlockingQueue<AMSchedulerEvent>();
public TaskSchedulerEventHandler(AppContext appContext,
- ClientService clientService) {
+ DAGClientServer clientService) {
super(TaskSchedulerEventHandler.class.getName());
this.appContext = appContext;
- eventHandler = appContext.getEventHandler();
- InetSocketAddress serviceAddr = clientService.getBindAddress();
- taskScheduler =
- new TaskScheduler(appContext.getApplicationAttemptId(),
- this,
- serviceAddr.getHostName(),
- serviceAddr.getPort(),
- serviceAddr.getHostName() +
- ":" + clientService.getHttpPort());
+ this.eventHandler = appContext.getEventHandler();
+ this.clientService = clientService;
}
public Map<ApplicationAccessType, String> getApplicationAcls() {
@@ -346,14 +339,22 @@ public class TaskSchedulerEventHandler extends AbstractService
@Override
public synchronized void init(Configuration conf) {
super.init(conf);
- taskScheduler.init(conf);
- // todo set heartbeat value from conf here
}
@Override
public synchronized void start() {
// FIXME hack alert how is this supposed to support multiple DAGs?
// Answer: this is shared across dags. need job==app-dag-master
+ // TODO set heartbeat value from conf here
+ InetSocketAddress serviceAddr = clientService.getBindAddress();
+ taskScheduler =
+ new TaskScheduler(appContext.getApplicationAttemptId(),
+ this,
+ serviceAddr.getHostName(),
+ serviceAddr.getPort(),
+ "");
+ taskScheduler.init(getConfig());
+
dagAppMaster = appContext.getAppMaster();
taskScheduler.start();
this.eventHandlingThread = new Thread() {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
index 9cdc845..44ccc07 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
@@ -18,34 +18,47 @@
package org.apache.tez.mapreduce;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.v2.LogParams;
import org.apache.hadoop.mapreduce.TaskCompletionEvent;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezRemoteException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
import java.io.IOException;
public class ClientServiceDelegate {
+ private static final Log LOG = LogFactory.getLog(ClientServiceDelegate.class);
- private final Configuration conf;
+ private final TezConfiguration conf;
private final ResourceMgrDelegate rm;
+ private DAGClient dagClient;
+ private String currentDAGId;
+ private TezClient tezClient;
+ private ApplicationReport appReport;
// FIXME
// how to handle completed jobs that the RM does not know about?
public ClientServiceDelegate(Configuration conf, ResourceMgrDelegate rm,
JobID jobId) {
- this.conf = new Configuration(conf); // Cloning for modifying.
+ this.conf = new TezConfiguration(conf); // Cloning for modifying.
// For faster redirects from AM to HS.
this.conf.setInt(
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
this.conf.getInt(MRJobConfig.MR_CLIENT_TO_AM_IPC_MAX_RETRIES,
MRJobConfig.DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES));
this.rm = rm;
+ tezClient = new TezClient(this.conf);
}
public org.apache.hadoop.mapreduce.Counters getJobCounters(JobID jobId)
@@ -74,17 +87,42 @@ public class ClientServiceDelegate {
}
public JobStatus getJobStatus(JobID oldJobID) throws IOException {
- org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
- TypeConverter.toYarn(oldJobID);
- ApplicationReport appReport;
try {
- appReport = rm.getApplicationReport(jobId.getAppId());
- } catch (YarnRemoteException e) {
- throw new IOException(e);
+ if(dagClient == null) {
+ appReport = getAppReport(oldJobID);
+ if(appReport.getYarnApplicationState() != YarnApplicationState.RUNNING) {
+ // if job not running return status from appReport;
+ return getJobStatusFromRM(appReport);
+ } else {
+ // job is running. create dag am client
+ dagClient = tezClient.getDAGClient(appReport.getHost(),
+ appReport.getRpcPort());
+ currentDAGId = dagClient.getAllDAGs().get(0);
+ }
+ }
+ // return status from client. use saved appReport for queue etc
+ DAGStatus dagStatus = dagClient.getDAGStatus(currentDAGId);
+ return new DAGJobStatus(appReport, dagStatus);
+ } catch (TezRemoteException e) {
+ // AM not responding
+ dagClient = null;
+ currentDAGId = null;
+ appReport = getAppReport(oldJobID);
+ if(appReport.getYarnApplicationState() != YarnApplicationState.RUNNING) {
+ LOG.info("App not running. Falling back to RM for report.");
+ } else {
+ LOG.warn("App running but failed to get report from AM.", e);
+ }
}
+
+ // final fallback
+ return getJobStatusFromRM(appReport);
+ }
+
+ private JobStatus getJobStatusFromRM(ApplicationReport appReport) {
JobStatus jobStatus =
- new DAGJobStatus(appReport);
- return jobStatus;
+ new DAGJobStatus(appReport, null);
+ return jobStatus;
}
public org.apache.hadoop.mapreduce.TaskReport[] getTaskReports(
@@ -114,4 +152,14 @@ public class ClientServiceDelegate {
// FIXME logs for an attempt?
throw new UnsupportedOperationException();
}
+
+ private ApplicationReport getAppReport(JobID oldJobID) throws IOException {
+ org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
+ TypeConverter.toYarn(oldJobID);
+ try {
+ return rm.getApplicationReport(jobId.getAppId());
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef37823c/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
index d58cea4..08c61f8 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
@@ -32,14 +32,20 @@ import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.Progress;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+import org.mortbay.log.Log;
public class DAGJobStatus extends JobStatus {
private final ApplicationReport report;
+ private final DAGStatus dagStatus;
- public DAGJobStatus(ApplicationReport appReport) {
+ public DAGJobStatus(ApplicationReport appReport, DAGStatus dagStatus) {
super();
this.report = appReport;
+ this.dagStatus = dagStatus;
}
@Override
@@ -129,6 +135,9 @@ public class DAGJobStatus extends JobStatus {
@Override
public synchronized float getMapProgress() {
+ if(dagStatus != null) {
+ return getProgress(MultiStageMRConfigUtil.getInitialMapVertexName());
+ }
if (report.getYarnApplicationState().equals(
YarnApplicationState.FINISHED)
&& report.getFinalApplicationStatus().equals(
@@ -160,6 +169,9 @@ public class DAGJobStatus extends JobStatus {
@Override
public synchronized float getReduceProgress() {
+ if(dagStatus != null) {
+ return getProgress(MultiStageMRConfigUtil.getFinalReduceVertexName());
+ }
if (report.getYarnApplicationState().equals(
YarnApplicationState.FINISHED)
&& report.getFinalApplicationStatus().equals(
@@ -350,5 +362,18 @@ public class DAGJobStatus extends JobStatus {
buffer.append("needed-mem" + getNeededMem());
return buffer.toString();
}
+
+ private float getProgress(String vertexName) {
+ Progress progress = dagStatus.getVertexProgress().get(vertexName);
+ if(progress == null) {
+ // no such stage. return 0 like MR app currently does.
+ return 0;
+ }
+ float totalTasks = (float) progress.getTotalTaskCount();
+ if(totalTasks != 0) {
+ return progress.getSucceededTaskCount()/totalTasks;
+ }
+ return 1;
+ }
}
[33/50] [abbrv] git commit: TEZ-118. Create a MiniTezCluster for
allowing of unit tests.
Posted by ss...@apache.org.
TEZ-118. Create a MiniTezCluster for allowing of unit tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/2340884c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/2340884c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/2340884c
Branch: refs/heads/master
Commit: 2340884c3ce85364ee1892909a58b82f532560db
Parents: 82ee45e
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed May 29 13:32:47 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Wed May 29 13:32:47 2013 -0700
----------------------------------------------------------------------
pom.xml | 26 +
.../main/java/org/apache/tez/dag/api/Vertex.java | 41 +-
tez-dag/pom.xml | 22 -
tez-dag/src/test/resources/krb5.conf | 28 -
.../tez/mapreduce/examples/ExampleDriver.java | 16 +-
.../apache/tez/mapreduce/examples/MRRSleepJob.java | 460 +++++++++++++++
tez-mapreduce-tests/pom.xml | 100 ++++
.../apache/tez/mapreduce/MiniMRRTezCluster.java | 151 +++++
.../java/org/apache/tez/mapreduce/TestMRRJobs.java | 344 +++++++++++
tez-mapreduce/pom.xml | 12 +
.../java/org/apache/tez/mapreduce/YARNRunner.java | 108 ++--
11 files changed, 1172 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7fa336a..0e4f8e7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,6 +106,16 @@
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
+ <artifactId>tez-mapreduce-examples</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-mapreduce-tests</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
<artifactId>tez-dag</artifactId>
<version>${project.version}</version>
</dependency>
@@ -191,6 +201,19 @@
<version>${hadoop.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.5</version>
@@ -264,6 +287,7 @@
<module>tez-yarn-client</module>
<module>tez-mapreduce</module>
<module>tez-mapreduce-examples</module>
+ <module>tez-mapreduce-tests</module>
<module>tez-dag</module>
<module>tez-dist</module>
</modules>
@@ -299,7 +323,9 @@
<forkMode>always</forkMode>
<forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
<argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
<environmentVariables>
+ <JAVA_HOME>${java.home}</JAVA_HOME>
<MALLOC_ARENA_MAX>4</MALLOC_ARENA_MAX>
</environmentVariables>
<systemPropertyVariables>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java
index 6bea664..cbcac16 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/Vertex.java
@@ -30,30 +30,30 @@ public class Vertex { // FIXME rename to Task
private final String vertexName;
private final String processorName;
-
+
private final int parallelism;
private VertexLocationHint taskLocationsHint;
private Resource taskResource;
private Map<String, LocalResource> taskLocalResources;
private Map<String, String> taskEnvironment;
-
+
private final List<Vertex> inputVertices = new ArrayList<Vertex>();
private final List<Vertex> outputVertices = new ArrayList<Vertex>();
private final List<String> inputEdgeIds = new ArrayList<String>();
private final List<String> outputEdgeIds = new ArrayList<String>();
private String javaOpts = "";
-
-
+
+
public Vertex(String vertexName, String processorName, int parallelism) {
this.vertexName = vertexName;
this.processorName = processorName;
this.parallelism = parallelism;
}
-
+
public String getVertexName() { // FIXME rename to getName()
return vertexName;
}
-
+
public String getProcessorName() {
return processorName;
}
@@ -61,16 +61,19 @@ public class Vertex { // FIXME rename to Task
public int getParallelism() {
return parallelism;
}
-
+
public void setTaskResource(Resource resource) {
this.taskResource = resource;
}
-
+
public Resource getTaskResource() {
return taskResource;
}
-
+
public void setTaskLocationsHint(TaskLocationHint[] locations) {
+ if (locations == null) {
+ return;
+ }
assert locations.length == parallelism;
taskLocationsHint = new VertexLocationHint(parallelism, locations);
}
@@ -79,31 +82,31 @@ public class Vertex { // FIXME rename to Task
VertexLocationHint getTaskLocationsHint() {
return taskLocationsHint;
}
-
+
public void setTaskLocalResources(Map<String, LocalResource> localResources) {
this.taskLocalResources = localResources;
}
-
+
public Map<String, LocalResource> getTaskLocalResources() {
return taskLocalResources;
}
-
+
public void setTaskEnvironment(Map<String, String> environment) {
this.taskEnvironment = environment;
}
-
+
public Map<String, String> getTaskEnvironment() {
return taskEnvironment;
}
-
+
public void setJavaOpts(String javaOpts){
this. javaOpts = javaOpts;
}
-
+
public String getJavaOpts(){
return javaOpts;
}
-
+
@Override
public String toString() {
return "[" + vertexName + " : " + processorName + "]";
@@ -118,15 +121,15 @@ public class Vertex { // FIXME rename to Task
outputVertices.add(outputVertex);
outputEdgeIds.add(edgeId);
}
-
+
List<Vertex> getInputVertices() {
return inputVertices;
}
-
+
List<Vertex> getOutputVertices() {
return outputVertices;
}
-
+
List<String> getInputEdgeIds() {
return inputEdgeIds;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-dag/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index fc03bea..34194db 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -22,8 +22,6 @@
<version>0.2.0-SNAPSHOT</version>
</parent>
<properties>
- <maven.test.redirectTestOutputToFile>true
- </maven.test.redirectTestOutputToFile>
<tez.component>tez-dag</tez.component>
</properties>
<artifactId>tez-dag</artifactId>
@@ -102,26 +100,6 @@
</dependencies>
<build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <forkMode>always</forkMode>
- <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
- <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
- <environmentVariables>
- <MALLOC_ARENA_MAX>4</MALLOC_ARENA_MAX>
- </environmentVariables>
- <systemPropertyVariables>
- <java.security.krb5.conf>${basedir}/src/test/resources/krb5.conf
- </java.security.krb5.conf>
- </systemPropertyVariables>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-dag/src/test/resources/krb5.conf
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/resources/krb5.conf b/tez-dag/src/test/resources/krb5.conf
deleted file mode 100644
index 121ac6d..0000000
--- a/tez-dag/src/test/resources/krb5.conf
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-[libdefaults]
- default_realm = APACHE.ORG
- udp_preference_limit = 1
- extra_addresses = 127.0.0.1
-[realms]
- APACHE.ORG = {
- admin_server = localhost:88
- kdc = localhost:88
- }
-[domain_realm]
- localhost = APACHE.ORG
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
index 94ba79f..468e0ac 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
@@ -24,23 +24,23 @@ import org.apache.tez.mapreduce.examples.terasort.TeraSort;
import org.apache.tez.mapreduce.examples.terasort.TeraValidate;
/**
- * A description of an example program based on its class and a
+ * A description of an example program based on its class and a
* human-readable description.
*/
public class ExampleDriver {
-
+
public static void main(String argv[]){
int exitCode = -1;
ProgramDriver pgd = new ProgramDriver();
try {
- pgd.addClass("wordcount", WordCount.class,
+ pgd.addClass("wordcount", WordCount.class,
"A map/reduce program that counts the words in the input files.");
- pgd.addClass("wordcountmrrtest", WordCountMRRTest.class,
+ pgd.addClass("wordcountmrrtest", WordCountMRRTest.class,
"A map/reduce program that counts the words in the input files."
+ " Map splits on spaces. First reduce splits on \".\"");
- pgd.addClass("randomwriter", RandomWriter.class,
+ pgd.addClass("randomwriter", RandomWriter.class,
"A map/reduce program that writes 10GB of random data per node.");
- pgd.addClass("randomtextwriter", RandomTextWriter.class,
+ pgd.addClass("randomtextwriter", RandomTextWriter.class,
"A map/reduce program that writes 10GB of random textual data per node.");
pgd.addClass("sort", Sort.class,
"A map/reduce program that sorts the data written by the random"
@@ -61,12 +61,14 @@ public class ExampleDriver {
+ " containing employee_name department name per line of input"
+ " and generates count of employees per department and"
+ " sorted on employee count");
+ pgd.addClass("mrrsleep", MRRSleepJob.class,
+ "MRR Sleep Job");
exitCode = pgd.run(argv);
}
catch(Throwable e){
e.printStackTrace();
}
-
+
System.exit(exitCode);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
new file mode 100644
index 0000000..6115c6f
--- /dev/null
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -0,0 +1,460 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+
+/**
+ * Dummy class for testing MR framefork. Sleeps for a defined period
+ * of time in mapper and reducer. Generates fake input for map / reduce
+ * jobs. Note that generated number of input pairs is in the order
+ * of <code>numMappers * mapSleepTime / 100</code>, so the job uses
+ * some disk space.
+ */
+public class MRRSleepJob extends Configured implements Tool {
+
+ public static Log LOG = LogFactory.getLog(MRRSleepJob.class);
+
+ public static String MAP_SLEEP_COUNT = "mrr.sleepjob.map.sleep.count";
+ public static String REDUCE_SLEEP_COUNT =
+ "mrr.sleepjob.reduce.sleep.count";
+ public static String MAP_SLEEP_TIME = "mrr.sleepjob.map.sleep.time";
+ public static String REDUCE_SLEEP_TIME =
+ "mrr.sleepjob.reduce.sleep.time";
+ public static String IREDUCE_SLEEP_COUNT =
+ "mrr.sleepjob.ireduce.sleep.count";
+ public static String IREDUCE_SLEEP_TIME =
+ "mrr.sleepjob.ireduce.sleep.time";
+ public static String IREDUCE_STAGES_COUNT =
+ "mrr.sleepjob.ireduces.stages.count";
+
+ // Flags to inject failures
+ public static String MAP_THROW_ERROR = "mrr.sleepjob.map.throw.error";
+ public static String MAP_FATAL_ERROR = "mrr.sleepjob.map.fatal.error";
+ public static String MAP_ERROR_TASK_IDS = "mrr.sleepjob.map.error.task.ids";
+
+ public static class MRRSleepJobPartitioner extends
+ Partitioner<IntWritable, IntWritable> {
+ public int getPartition(IntWritable k, IntWritable v, int numPartitions) {
+ return k.get() % numPartitions;
+ }
+ }
+
+ public static class EmptySplit extends InputSplit implements Writable {
+ public void write(DataOutput out) throws IOException { }
+ public void readFields(DataInput in) throws IOException { }
+ public long getLength() { return 0L; }
+ public String[] getLocations() { return new String[0]; }
+ }
+
+ public static class SleepInputFormat
+ extends InputFormat<IntWritable,IntWritable> {
+
+ public List<InputSplit> getSplits(JobContext jobContext) {
+ List<InputSplit> ret = new ArrayList<InputSplit>();
+ int numSplits = jobContext.getConfiguration().
+ getInt(MRJobConfig.NUM_MAPS, 1);
+ for (int i = 0; i < numSplits; ++i) {
+ ret.add(new EmptySplit());
+ }
+ return ret;
+ }
+
+ public RecordReader<IntWritable,IntWritable> createRecordReader(
+ InputSplit ignored, TaskAttemptContext taskContext)
+ throws IOException {
+ Configuration conf = taskContext.getConfiguration();
+ String vertexName = conf.get(
+ org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
+ boolean isIntermediateReduce =
+ MultiStageMRConfigUtil.getIntermediateStageNum(vertexName) != -1;
+
+ final int count =
+ (isIntermediateReduce)?
+ conf.getInt(IREDUCE_SLEEP_COUNT, 1) :
+ conf.getInt(MAP_SLEEP_COUNT, 1);
+ if (count < 0) throw new IOException("Invalid map count: " + count);
+
+ int totalIReduces = MultiStageMRConfigUtil.getNumIntermediateStages(conf);
+ boolean finalIReduce = totalIReduces ==
+ (MultiStageMRConfigUtil.getIntermediateStageNum(vertexName) + 1);
+
+ final int redcount = finalIReduce?
+ conf.getInt(REDUCE_SLEEP_COUNT, 1) :
+ conf.getInt(IREDUCE_SLEEP_COUNT, 1);
+ if (redcount < 0)
+ throw new IOException("Invalid reduce count: " + redcount);
+
+ final int emitPerMapTask = (redcount * taskContext.getNumReduceTasks());
+
+ return new RecordReader<IntWritable,IntWritable>() {
+ private int records = 0;
+ private int emitCount = 0;
+ private IntWritable key = null;
+ private IntWritable value = null;
+
+ public void initialize(InputSplit split, TaskAttemptContext context) {
+ }
+
+ public boolean nextKeyValue()
+ throws IOException {
+ if (count == 0) {
+ return false;
+ }
+ key = new IntWritable();
+ key.set(emitCount);
+ int emit = emitPerMapTask / count;
+ if ((emitPerMapTask) % count > records) {
+ ++emit;
+ }
+ emitCount += emit;
+ value = new IntWritable();
+ value.set(emit);
+ return records++ < count;
+ }
+ public IntWritable getCurrentKey() { return key; }
+ public IntWritable getCurrentValue() { return value; }
+ public void close() throws IOException { }
+ public float getProgress() throws IOException {
+ return count == 0 ? 100 : records / ((float)count);
+ }
+ };
+ }
+ }
+
+ public static class SleepMapper
+ extends Mapper<IntWritable, IntWritable, IntWritable, IntWritable> {
+ private long mapSleepDuration = 100;
+ private int mapSleepCount = 1;
+ private int count = 0;
+ private String vertexName;
+ private boolean throwError = false;
+ private boolean throwFatal = false;
+ private boolean finalAttempt = false;
+
+ protected void setup(Context context)
+ throws IOException, InterruptedException {
+ Configuration conf = context.getConfiguration();
+ this.mapSleepCount =
+ conf.getInt(MAP_SLEEP_COUNT, mapSleepCount);
+ this.mapSleepDuration = mapSleepCount == 0 ? 0 :
+ conf.getLong(MAP_SLEEP_TIME , 100) / mapSleepCount;
+ vertexName = conf.get(
+ org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
+
+ TaskAttemptID taId = context.getTaskAttemptID();
+ String[] taskIds = conf.getStrings(MAP_ERROR_TASK_IDS);
+ if (taId.getId()+1 >= context.getMaxMapAttempts()) {
+ finalAttempt = true;
+ }
+ boolean found = false;
+ if (taskIds != null) {
+ if (taskIds.length == 1 && taskIds[0].equals("*")) {
+ found = true;
+ }
+ if (!found) {
+ for (String taskId : taskIds) {
+ if (Integer.valueOf(taskId).intValue() ==
+ taId.getTaskID().getId()) {
+ found = true;
+ break;
+ }
+ }
+ }
+ }
+ if (found) {
+ if (!finalAttempt) {
+ throwError = conf.getBoolean(MAP_THROW_ERROR, false);
+ }
+ throwFatal = conf.getBoolean(MAP_FATAL_ERROR, false);
+ }
+ }
+
+ public void map(IntWritable key, IntWritable value, Context context
+ ) throws IOException, InterruptedException {
+ //it is expected that every map processes mapSleepCount number of records.
+ try {
+ LOG.info("Sleeping in InitialMap"
+ + ", vertexName=" + vertexName
+ + ", taskAttemptId=" + context.getTaskAttemptID()
+ + ", mapSleepDuration=" + mapSleepDuration
+ + ", mapSleepCount=" + mapSleepCount
+ + ", sleepLeft="
+ + (mapSleepDuration * (mapSleepCount - count)));
+ context.setStatus("Sleeping... (" +
+ (mapSleepDuration * (mapSleepCount - count)) + ") ms left");
+ Thread.sleep(mapSleepDuration);
+ if (throwError || throwFatal) {
+ throw new IOException("Throwing a simulated error from map");
+ }
+ }
+ catch (InterruptedException ex) {
+ throw (IOException)new IOException(
+ "Interrupted while sleeping").initCause(ex);
+ }
+ ++count;
+ // output reduceSleepCount * numReduce number of random values, so that
+ // each reducer will get reduceSleepCount number of keys.
+ int k = key.get();
+ for (int i = 0; i < value.get(); ++i) {
+ context.write(new IntWritable(k + i), new IntWritable(1));
+ }
+ }
+ }
+
+ public static class ISleepReducer
+ extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
+ private long iReduceSleepDuration = 100;
+ private int iReduceSleepCount = 1;
+ private int count = 0;
+ private String vertexName;
+
+ protected void setup(Context context)
+ throws IOException, InterruptedException {
+ Configuration conf = context.getConfiguration();
+ this.iReduceSleepCount =
+ conf.getInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
+ this.iReduceSleepDuration = iReduceSleepCount == 0 ? 0 :
+ conf.getLong(IREDUCE_SLEEP_TIME , 100) / iReduceSleepCount;
+ vertexName = conf.get(
+ org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
+ }
+
+ public void reduce(IntWritable key, Iterable<IntWritable> values,
+ Context context)
+ throws IOException, InterruptedException {
+ try {
+ LOG.info("Sleeping in IntermediateReduce"
+ + ", vertexName=" + vertexName
+ + ", taskAttemptId=" + context.getTaskAttemptID()
+ + ", iReduceSleepDuration=" + iReduceSleepDuration
+ + ", iReduceSleepCount=" + iReduceSleepCount
+ + ", sleepLeft="
+ + (iReduceSleepDuration * (iReduceSleepCount - count)));
+ context.setStatus("Sleeping... (" +
+ (iReduceSleepDuration * (iReduceSleepCount - count)) + ") ms left");
+ Thread.sleep(iReduceSleepDuration);
+ }
+ catch (InterruptedException ex) {
+ throw (IOException)new IOException(
+ "Interrupted while sleeping").initCause(ex);
+ }
+ ++count;
+ // output reduceSleepCount * numReduce number of random values, so that
+ // each reducer will get reduceSleepCount number of keys.
+ int k = key.get();
+ for (IntWritable value : values) {
+ for (int i = 0; i < value.get(); ++i) {
+ context.write(new IntWritable(k + i), new IntWritable(1));
+ }
+ }
+ }
+ }
+
+ public static class SleepReducer
+ extends Reducer<IntWritable, IntWritable, NullWritable, NullWritable> {
+ private long reduceSleepDuration = 100;
+ private int reduceSleepCount = 1;
+ private int count = 0;
+ private String vertexName;
+
+ protected void setup(Context context)
+ throws IOException, InterruptedException {
+ Configuration conf = context.getConfiguration();
+ this.reduceSleepCount =
+ conf.getInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
+ this.reduceSleepDuration = reduceSleepCount == 0 ? 0 :
+ conf.getLong(REDUCE_SLEEP_TIME , 100) / reduceSleepCount;
+ vertexName = conf.get(
+ org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
+ }
+
+ public void reduce(IntWritable key, Iterable<IntWritable> values,
+ Context context)
+ throws IOException {
+ try {
+ LOG.info("Sleeping in FinalReduce"
+ + ", vertexName=" + vertexName
+ + ", taskAttemptId=" + context.getTaskAttemptID()
+ + ", reduceSleepDuration=" + reduceSleepDuration
+ + ", reduceSleepCount=" + reduceSleepCount
+ + ", sleepLeft="
+ + (reduceSleepDuration * (reduceSleepCount - count)));
+ context.setStatus("Sleeping... (" +
+ (reduceSleepDuration * (reduceSleepCount - count)) + ") ms left");
+ Thread.sleep(reduceSleepDuration);
+
+ }
+ catch (InterruptedException ex) {
+ throw (IOException)new IOException(
+ "Interrupted while sleeping").initCause(ex);
+ }
+ count++;
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new MRRSleepJob(), args);
+ System.exit(res);
+ }
+
+ public Job createJob(int numMapper, int numReducer, int iReduceStagesCount,
+ int numIReducer, long mapSleepTime, int mapSleepCount,
+ long reduceSleepTime, int reduceSleepCount,
+ long iReduceSleepTime, int iReduceSleepCount)
+ throws IOException {
+ Configuration conf = getConf();
+ conf.setLong(MAP_SLEEP_TIME, mapSleepTime);
+ conf.setLong(REDUCE_SLEEP_TIME, reduceSleepTime);
+ conf.setLong(IREDUCE_SLEEP_TIME, iReduceSleepTime);
+ conf.setInt(MAP_SLEEP_COUNT, mapSleepCount);
+ conf.setInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
+ conf.setInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
+ conf.setInt(MRJobConfig.NUM_MAPS, numMapper);
+ conf.setInt(IREDUCE_STAGES_COUNT, iReduceStagesCount);
+
+ // Configure intermediate reduces
+ conf.setInt(
+ org.apache.tez.mapreduce.hadoop.MRJobConfig.MRR_INTERMEDIATE_STAGES,
+ iReduceStagesCount);
+ LOG.info("Running MRR with " + iReduceStagesCount + " IR stages");
+
+ for (int i = 0; i < iReduceStagesCount; ++i) {
+ // Set reducer class for intermediate reduce
+ conf.setClass(
+ MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+ "mapreduce.job.reduce.class"), ISleepReducer.class, Reducer.class);
+ // Set reducer output key class
+ conf.setClass(
+ MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+ "mapreduce.map.output.key.class"), IntWritable.class, Object.class);
+ // Set reducer output value class
+ conf.setClass(
+ MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+ "mapreduce.map.output.value.class"), IntWritable.class, Object.class);
+ conf.setInt(
+ MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+ "mapreduce.job.reduces"), numIReducer);
+ }
+
+ Job job = Job.getInstance(conf, "sleep");
+ job.setNumReduceTasks(numReducer);
+ job.setJarByClass(MRRSleepJob.class);
+ job.setNumReduceTasks(numReducer);
+ job.setMapperClass(SleepMapper.class);
+ job.setMapOutputKeyClass(IntWritable.class);
+ job.setMapOutputValueClass(IntWritable.class);
+ job.setReducerClass(SleepReducer.class);
+ job.setOutputFormatClass(NullOutputFormat.class);
+ job.setInputFormatClass(SleepInputFormat.class);
+ job.setPartitionerClass(MRRSleepJobPartitioner.class);
+ job.setSpeculativeExecution(false);
+ job.setJobName("Sleep job");
+
+
+ FileInputFormat.addInputPath(job, new Path("ignored"));
+ return job;
+ }
+
+ public int run(String[] args) throws Exception {
+
+ if(args.length < 1) {
+ System.err.println("MRRSleepJob [-m numMapper] [-r numReducer]" +
+ " [-ir numIntermediateReducer]" +
+ " [-irs numIntermediateReducerStages]" +
+ " [-mt mapSleepTime (msec)] [-rt reduceSleepTime (msec)]" +
+ " [-irt intermediateReduceSleepTime]" +
+ " [-recordt recordSleepTime (msec)]");
+ ToolRunner.printGenericCommandUsage(System.err);
+ return 2;
+ }
+
+ int numMapper = 1, numReducer = 1, numIReducer = 1;
+ long mapSleepTime = 100, reduceSleepTime = 100, recSleepTime = 100,
+ iReduceSleepTime=1;
+ int mapSleepCount = 1, reduceSleepCount = 1, iReduceSleepCount = 1;
+ int iReduceStagesCount = 1;
+
+ for(int i=0; i < args.length; i++ ) {
+ if(args[i].equals("-m")) {
+ numMapper = Integer.parseInt(args[++i]);
+ }
+ else if(args[i].equals("-r")) {
+ numReducer = Integer.parseInt(args[++i]);
+ }
+ else if(args[i].equals("-ir")) {
+ numIReducer = Integer.parseInt(args[++i]);
+ }
+ else if(args[i].equals("-mt")) {
+ mapSleepTime = Long.parseLong(args[++i]);
+ }
+ else if(args[i].equals("-rt")) {
+ reduceSleepTime = Long.parseLong(args[++i]);
+ }
+ else if(args[i].equals("-irt")) {
+ iReduceSleepTime = Long.parseLong(args[++i]);
+ }
+ else if(args[i].equals("-irs")) {
+ iReduceStagesCount = Integer.parseInt(args[++i]);
+ }
+ else if (args[i].equals("-recordt")) {
+ recSleepTime = Long.parseLong(args[++i]);
+ }
+ }
+
+ // sleep for *SleepTime duration in Task by recSleepTime per record
+ mapSleepCount = (int)Math.ceil(mapSleepTime / ((double)recSleepTime));
+ reduceSleepCount = (int)Math.ceil(reduceSleepTime / ((double)recSleepTime));
+ iReduceSleepCount = (int)Math.ceil(iReduceSleepTime / ((double)recSleepTime));
+ Job job = createJob(numMapper, numReducer, iReduceStagesCount, numIReducer,
+ mapSleepTime, mapSleepCount, reduceSleepTime, reduceSleepCount,
+ iReduceSleepTime, iReduceSleepCount);
+ return job.waitForCompletion(true) ? 0 : 1;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/pom.xml b/tez-mapreduce-tests/pom.xml
new file mode 100644
index 0000000..0fb1024
--- /dev/null
+++ b/tez-mapreduce-tests/pom.xml
@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>tez-mapreduce-tests</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-dag</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-mapreduce</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-yarn-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-mapreduce-examples</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
new file mode 100644
index 0000000..41a1cf3
--- /dev/null
+++ b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
@@ -0,0 +1,151 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.mapreduce;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.ShuffleHandler;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
+import org.apache.hadoop.util.JarFinder;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.hadoop.yarn.service.Service;
+import org.apache.tez.dag.app.DAGAppMaster;
+import org.apache.tez.mapreduce.hadoop.MRConfig;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+
+/**
+ * Configures and starts the Tez-specific components in the YARN cluster.
+ *
+ * When using this mini cluster, the user is expected to
+ */
+public class MiniMRRTezCluster extends MiniYARNCluster {
+
+ public static final String APPJAR = JarFinder.getJar(DAGAppMaster.class);
+
+ private static final Log LOG = LogFactory.getLog(MiniMRRTezCluster.class);
+
+ private static final String YARN_CLUSTER_CONFIG = "yarn-site.xml";
+
+ private Path confFilePath;
+
+ public MiniMRRTezCluster(String testName) {
+ this(testName, 1);
+ }
+
+ public MiniMRRTezCluster(String testName, int noOfNMs) {
+ super(testName, noOfNMs, 4, 4);
+ }
+
+ public MiniMRRTezCluster(String testName, int noOfNMs,
+ int numLocalDirs, int numLogDirs) {
+ super(testName, noOfNMs, numLocalDirs, numLogDirs);
+ }
+
+ @Override
+ public void init(Configuration conf) {
+ conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_TEZ_FRAMEWORK_NAME);
+ if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
+ conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(),
+ "apps_staging_dir" + Path.SEPARATOR).getAbsolutePath());
+ }
+
+ // VMEM monitoring disabled, PMEM monitoring enabled.
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
+
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
+
+ try {
+ Path stagingPath = FileContext.getFileContext(conf).makeQualified(
+ new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
+ FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf);
+ if (fc.util().exists(stagingPath)) {
+ LOG.info(stagingPath + " exists! deleting...");
+ fc.delete(stagingPath, true);
+ }
+ LOG.info("mkdir: " + stagingPath);
+ fc.mkdir(stagingPath, null, true);
+
+ //mkdir done directory as well
+ String doneDir =
+ JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
+ Path doneDirPath = fc.makeQualified(new Path(doneDir));
+ fc.mkdir(doneDirPath, null, true);
+ } catch (IOException e) {
+ throw new YarnException("Could not create staging directory. ", e);
+ }
+ conf.set(MRConfig.MASTER_ADDRESS, "test");
+
+ //configure the shuffle service in NM
+ conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
+ new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
+ conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
+ ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
+ Service.class);
+
+ // Non-standard shuffle port
+ conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+
+ conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
+ DefaultContainerExecutor.class, ContainerExecutor.class);
+
+ // TestMRJobs is for testing non-uberized operation only; see TestUberAM
+ // for corresponding uberized tests.
+ conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
+
+ super.init(conf);
+ }
+
+ @Override
+ public void start() {
+ super.start();
+ File workDir = super.getTestWorkDir();
+ Configuration conf = super.getConfig();
+
+ confFilePath = new Path(workDir.getAbsolutePath(), YARN_CLUSTER_CONFIG);
+ File confFile = new File(confFilePath.toString());
+ try {
+ confFile.createNewFile();
+ conf.writeXml(new FileOutputStream(confFile));
+ confFile.deleteOnExit();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ confFilePath = new Path(confFile.getAbsolutePath());
+ }
+
+ public Path getConfigFilePath() {
+ return confFilePath;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java
new file mode 100644
index 0000000..520a1ac
--- /dev/null
+++ b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java
@@ -0,0 +1,344 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.mapreduce;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.RandomTextWriterJob;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobCounter;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskReport;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.JarFinder;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.tez.mapreduce.examples.MRRSleepJob;
+import org.apache.tez.mapreduce.hadoop.MRConfig;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestMRRJobs {
+
+ private static final Log LOG = LogFactory.getLog(TestMRRJobs.class);
+
+ protected static MiniMRRTezCluster mrrTezCluster;
+ protected static MiniDFSCluster dfsCluster;
+
+ private static Configuration conf = new Configuration();
+ private static FileSystem localFs;
+ private static FileSystem remoteFs;
+ static {
+ try {
+ localFs = FileSystem.getLocal(conf);
+ } catch (IOException io) {
+ throw new RuntimeException("problem getting local fs", io);
+ }
+ }
+
+ private static String TEST_ROOT_DIR = "target"
+ + Path.SEPARATOR + TestMRRJobs.class.getName() + "-tmpDir";
+
+ private static Path TEST_ROOT_DIR_PATH =
+ localFs.makeQualified(new Path(TEST_ROOT_DIR));
+ static Path APP_JAR = new Path(TEST_ROOT_DIR_PATH, "MRAppJar.jar");
+ static Path YARN_SITE_XML = new Path(TEST_ROOT_DIR_PATH, "yarn-site.xml");
+ private static final String OUTPUT_ROOT_DIR = "/tmp" + Path.SEPARATOR +
+ TestMRRJobs.class.getSimpleName();
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ try {
+ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
+ dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
+ .format(true).racks(null).build();
+ remoteFs = dfsCluster.getFileSystem();
+ } catch (IOException io) {
+ throw new RuntimeException("problem starting mini dfs cluster", io);
+ }
+
+ if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+ LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
+ + " not found. Not running test.");
+ return;
+ }
+
+ if (mrrTezCluster == null) {
+ mrrTezCluster = new MiniMRRTezCluster(TestMRRJobs.class.getName(), 1,
+ 1, 1);
+ Configuration conf = new Configuration();
+ conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
+ conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir");
+ mrrTezCluster.init(conf);
+ mrrTezCluster.start();
+ }
+
+ localFs.copyFromLocalFile(new Path(MiniMRRTezCluster.APPJAR), APP_JAR);
+ localFs.setPermission(APP_JAR, new FsPermission("700"));
+ localFs.copyFromLocalFile(mrrTezCluster.getConfigFilePath(), YARN_SITE_XML);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ if (mrrTezCluster != null) {
+ mrrTezCluster.stop();
+ mrrTezCluster = null;
+ }
+ if (dfsCluster != null) {
+ dfsCluster.shutdown();
+ dfsCluster = null;
+ }
+ }
+
+ @Test (timeout = 300000)
+ public void testMRRSleepJob() throws IOException, InterruptedException,
+ ClassNotFoundException {
+ LOG.info("\n\n\nStarting testMRRSleepJob().");
+
+ if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+ LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
+ + " not found. Not running test.");
+ return;
+ }
+
+ Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());
+
+ MRRSleepJob sleepJob = new MRRSleepJob();
+ sleepJob.setConf(sleepConf);
+
+ Job job = sleepJob.createJob(1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1);
+
+ job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+ job.addFileToClassPath(YARN_SITE_XML);
+ job.setJarByClass(MRRSleepJob.class);
+ job.setMaxMapAttempts(1); // speed up failures
+ job.submit();
+ String trackingUrl = job.getTrackingURL();
+ String jobId = job.getJobID().toString();
+ boolean succeeded = job.waitForCompletion(true);
+ Assert.assertTrue(succeeded);
+ Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
+ Assert.assertTrue("Tracking URL was " + trackingUrl +
+ " but didn't Match Job ID " + jobId ,
+ trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));
+
+ // FIXME once counters and task progress can be obtained properly
+ // TODO use dag client to test counters and task progress?
+ // what about completed jobs?
+
+ }
+
+ @Test (timeout = 60000)
+ public void testRandomWriter() throws IOException, InterruptedException,
+ ClassNotFoundException {
+
+ LOG.info("\n\n\nStarting testRandomWriter().");
+ if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+ LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
+ + " not found. Not running test.");
+ return;
+ }
+
+ RandomTextWriterJob randomWriterJob = new RandomTextWriterJob();
+ mrrTezCluster.getConfig().set(RandomTextWriterJob.TOTAL_BYTES, "3072");
+ mrrTezCluster.getConfig().set(RandomTextWriterJob.BYTES_PER_MAP, "1024");
+ Job job = randomWriterJob.createJob(mrrTezCluster.getConfig());
+ Path outputDir = new Path(OUTPUT_ROOT_DIR, "random-output");
+ FileOutputFormat.setOutputPath(job, outputDir);
+ job.setSpeculativeExecution(false);
+ job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+ job.addFileToClassPath(YARN_SITE_XML);
+ job.setJarByClass(RandomTextWriterJob.class);
+ job.setMaxMapAttempts(1); // speed up failures
+ job.submit();
+ String trackingUrl = job.getTrackingURL();
+ String jobId = job.getJobID().toString();
+ boolean succeeded = job.waitForCompletion(true);
+ Assert.assertTrue(succeeded);
+ Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
+ Assert.assertTrue("Tracking URL was " + trackingUrl +
+ " but didn't Match Job ID " + jobId ,
+ trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));
+
+ // Make sure there are three files in the output-dir
+
+ RemoteIterator<FileStatus> iterator =
+ FileContext.getFileContext(mrrTezCluster.getConfig()).listStatus(
+ outputDir);
+ int count = 0;
+ while (iterator.hasNext()) {
+ FileStatus file = iterator.next();
+ if (!file.getPath().getName()
+ .equals(FileOutputCommitter.SUCCEEDED_FILE_NAME)) {
+ count++;
+ }
+ }
+ Assert.assertEquals("Number of part files is wrong!", 3, count);
+
+ }
+
+
+ @Test (timeout = 60000)
+ public void testFailingJob() throws IOException, InterruptedException,
+ ClassNotFoundException {
+
+ LOG.info("\n\n\nStarting testFailingMapper().");
+
+ if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+ LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
+ + " not found. Not running test.");
+ return;
+ }
+
+ Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());
+
+ MRRSleepJob sleepJob = new MRRSleepJob();
+ sleepJob.setConf(sleepConf);
+
+ Job job = sleepJob.createJob(1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1);
+
+ job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+ job.addFileToClassPath(YARN_SITE_XML);
+ job.setJarByClass(MRRSleepJob.class);
+ job.setMaxMapAttempts(1); // speed up failures
+ job.getConfiguration().setBoolean(MRRSleepJob.MAP_FATAL_ERROR, true);
+ job.getConfiguration().set(MRRSleepJob.MAP_ERROR_TASK_IDS, "*");
+
+ job.submit();
+ boolean succeeded = job.waitForCompletion(true);
+ Assert.assertFalse(succeeded);
+ Assert.assertEquals(JobStatus.State.FAILED, job.getJobState());
+
+ // FIXME once counters and task progress can be obtained properly
+ // TODO verify failed task diagnostics
+ }
+
+ @Test (timeout = 300000)
+ public void testFailingAttempt() throws IOException, InterruptedException,
+ ClassNotFoundException {
+
+ LOG.info("\n\n\nStarting testFailingMapper().");
+
+ if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+ LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
+ + " not found. Not running test.");
+ return;
+ }
+
+ Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());
+
+ MRRSleepJob sleepJob = new MRRSleepJob();
+ sleepJob.setConf(sleepConf);
+
+ Job job = sleepJob.createJob(1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1);
+
+ job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+ job.addFileToClassPath(YARN_SITE_XML);
+ job.setJarByClass(MRRSleepJob.class);
+ job.setMaxMapAttempts(3); // speed up failures
+ job.getConfiguration().setBoolean(MRRSleepJob.MAP_THROW_ERROR, true);
+ job.getConfiguration().set(MRRSleepJob.MAP_ERROR_TASK_IDS, "0");
+
+ job.submit();
+ boolean succeeded = job.waitForCompletion(true);
+ Assert.assertTrue(succeeded);
+ Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
+
+ // FIXME once counters and task progress can be obtained properly
+ // TODO verify failed task diagnostics
+ }
+
+
+ /*
+ //@Test (timeout = 60000)
+ public void testMRRSleepJobWithSecurityOn() throws IOException,
+ InterruptedException, ClassNotFoundException {
+
+ LOG.info("\n\n\nStarting testMRRSleepJobWithSecurityOn().");
+
+ if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
+ return;
+ }
+
+ mrrTezCluster.getConfig().set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ mrrTezCluster.getConfig().set(YarnConfiguration.RM_KEYTAB, "/etc/krb5.keytab");
+ mrrTezCluster.getConfig().set(YarnConfiguration.NM_KEYTAB, "/etc/krb5.keytab");
+ mrrTezCluster.getConfig().set(YarnConfiguration.RM_PRINCIPAL,
+ "rm/sightbusy-lx@LOCALHOST");
+ mrrTezCluster.getConfig().set(YarnConfiguration.NM_PRINCIPAL,
+ "nm/sightbusy-lx@LOCALHOST");
+
+ UserGroupInformation.setConfiguration(mrrTezCluster.getConfig());
+
+ // Keep it in here instead of after RM/NM as multiple user logins happen in
+ // the same JVM.
+ UserGroupInformation user = UserGroupInformation.getCurrentUser();
+
+ LOG.info("User name is " + user.getUserName());
+ for (Token<? extends TokenIdentifier> str : user.getTokens()) {
+ LOG.info("Token is " + str.encodeToUrlString());
+ }
+ user.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ MRRSleepJob sleepJob = new MRRSleepJob();
+ sleepJob.setConf(mrrTezCluster.getConfig());
+ Job job = sleepJob.createJob(3, 0, 10000, 1, 0, 0);
+ // //Job with reduces
+ // Job job = sleepJob.createJob(3, 2, 10000, 1, 10000, 1);
+ job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+ job.submit();
+ String trackingUrl = job.getTrackingURL();
+ String jobId = job.getJobID().toString();
+ job.waitForCompletion(true);
+ Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
+ Assert.assertTrue("Tracking URL was " + trackingUrl +
+ " but didn't Match Job ID " + jobId ,
+ trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));
+ return null;
+ }
+ });
+
+ // TODO later: add explicit "isUber()" checks of some sort
+ }
+ */
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/tez-mapreduce/pom.xml b/tez-mapreduce/pom.xml
index e785f6b..e32b098 100644
--- a/tez-mapreduce/pom.xml
+++ b/tez-mapreduce/pom.xml
@@ -54,6 +54,18 @@
<artifactId>hadoop-mapreduce-client-common</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2340884c/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index a79a4f9..801ce8a 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -126,6 +126,7 @@ public class YARNRunner implements ClientProtocol {
final public static FsPermission DAG_FILE_PERMISSION =
FsPermission.createImmutable((short) 0644);
+ final public static int UTF8_CHUNK_SIZE = 16 * 1024;
/**
* Yarn runner incapsulates the client interface of
@@ -395,7 +396,7 @@ public class YARNRunner implements ClientProtocol {
}
}
-
+
private void setupMapReduceEnv(Configuration jobConf,
Map<String, String> environment, boolean isMap) throws IOException {
@@ -470,7 +471,7 @@ public class YARNRunner implements ClientProtocol {
+ "=" + entry.getValue());
}
}
-
+
private Vertex configureIntermediateReduceStage(FileSystem fs, JobID jobId,
Configuration jobConf, String jobSubmitDir, Credentials ts,
Map<String, LocalResource> jobLocalResources, int iReduceIndex)
@@ -482,7 +483,7 @@ public class YARNRunner implements ClientProtocol {
Vertex vertex = new Vertex(
MultiStageMRConfigUtil.getIntermediateStageVertexName(stageNum),
"org.apache.tez.mapreduce.processor.reduce.ReduceProcessor", numTasks);
-
+
Map<String, String> reduceEnv = new HashMap<String, String>();
setupMapReduceEnv(conf, reduceEnv, false);
@@ -501,7 +502,7 @@ public class YARNRunner implements ClientProtocol {
vertex.setTaskResource(reduceResource);
vertex.setJavaOpts(getReduceJavaOpts(conf));
-
+
return vertex;
}
@@ -518,7 +519,7 @@ public class YARNRunner implements ClientProtocol {
vertices[i] = configureIntermediateReduceStage(fs, jobId, jobConf, jobSubmitDir, ts,
jobLocalResources, i);
dag.addVertex(vertices[i]);
-
+
LOG.info("XXXX Adding intermediate vertex to DAG"
+ ", vertexName=" + vertices[i].getVertexName()
+ ", processor=" + vertices[i].getProcessorName()
@@ -550,7 +551,7 @@ public class YARNRunner implements ClientProtocol {
String mapProcessor = "org.apache.tez.mapreduce.processor.map.MapProcessor";
Vertex mapVertex = new Vertex(
MultiStageMRConfigUtil.getInitialMapVertexName(),
- mapProcessor, numMaps);
+ mapProcessor, numMaps);
// FIXME set up map environment
Map<String, String> mapEnv = new HashMap<String, String>();
@@ -575,7 +576,7 @@ public class YARNRunner implements ClientProtocol {
mapVertex.setTaskResource(mapResource);
mapVertex.setJavaOpts(getMapJavaOpts(jobConf));
-
+
LOG.info("XXXX Adding map vertex to DAG"
+ ", vertexName=" + mapVertex.getVertexName()
+ ", processor=" + mapVertex.getProcessorName()
@@ -618,7 +619,7 @@ public class YARNRunner implements ClientProtocol {
reduceVertex.setTaskResource(reduceResource);
reduceVertex.setJavaOpts(getReduceJavaOpts(jobConf));
-
+
LOG.info("XXXX Adding reduce vertex to DAG"
+ ", vertexName=" + reduceVertex.getVertexName()
+ ", processor=" + reduceVertex.getProcessorName()
@@ -677,11 +678,11 @@ public class YARNRunner implements ClientProtocol {
dag.addConfiguration(entry.getValue(), mrConf.get(entry.getKey()));
}
}
-
+
String jobName = mrConf.get(MRJobConfig.JOB_NAME);
if(jobName != null) {
dag.setName(jobName);
- }
+ }
}
private ApplicationSubmissionContext createApplicationSubmissionContext(
@@ -712,13 +713,13 @@ public class YARNRunner implements ClientProtocol {
vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
//[Debug AppMaster] Current simplest way to attach debugger to AppMaster
-// Uncomment the following, then launch a regular job, eg
-// >hadoop jar {path}\hadoop-mapreduce-examples-3.0.0-SNAPSHOT.jar pi 2 2
+// Uncomment the following, then launch a regular job, eg
+// >hadoop jar {path}\hadoop-mapreduce-examples-3.0.0-SNAPSHOT.jar pi 2 2
// LOG.error(" !!!!!!!!!");
// LOG.error(" !!!!!!!!! Launching AppMaster in debug/suspend mode. Attach to port 8002");
// LOG.error(" !!!!!!!!!");
// vargs.add("-Xdebug -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=8002,server=y,suspend=y");
-
+
// TODO -Dtez.root.logger??
String amLogLevel = jobConf.get(MRJobConfig.MR_AM_LOG_LEVEL,
MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
@@ -782,8 +783,10 @@ public class YARNRunner implements ClientProtocol {
DAGPlan dagPB = dag.createDag();
FSDataOutputStream dagPBOutBinaryStream = null;
FSDataOutputStream dagPBOutTextStream = null;
- Path binaryPath = new Path(jobSubmitDir, TezConfiguration.DAG_AM_PLAN_PB_BINARY);
- Path textPath = new Path(jobSubmitDir, TezConfiguration.DAG_AM_PLAN_PB_TEXT);
+ Path binaryPath = new Path(jobSubmitDir,
+ TezConfiguration.DAG_AM_PLAN_PB_BINARY);
+ Path textPath = new Path(jobSubmitDir,
+ TezConfiguration.DAG_AM_PLAN_PB_TEXT);
try {
//binary output
dagPBOutBinaryStream = FileSystem.create(fs, binaryPath,
@@ -793,7 +796,22 @@ public class YARNRunner implements ClientProtocol {
// text / human-readable output
dagPBOutTextStream = FileSystem.create(fs, textPath,
new FsPermission(DAG_FILE_PERMISSION));
- dagPBOutTextStream.writeUTF(dagPB.toString());
+ String dagPBStr = dagPB.toString();
+ int dagPBStrLen = dagPBStr.length();
+ if (dagPBStrLen <= UTF8_CHUNK_SIZE) {
+ dagPBOutTextStream.writeUTF(dagPBStr);
+ } else {
+ int startIndex = 0;
+ while (startIndex < dagPBStrLen) {
+ int endIndex = startIndex + UTF8_CHUNK_SIZE;
+ if (endIndex > dagPBStrLen) {
+ endIndex = dagPBStrLen;
+ }
+ dagPBOutTextStream.writeUTF(
+ dagPBStr.substring(startIndex, endIndex));
+ startIndex += UTF8_CHUNK_SIZE;
+ }
+ }
} finally {
if(dagPBOutBinaryStream != null){
dagPBOutBinaryStream.close();
@@ -823,7 +841,7 @@ public class YARNRunner implements ClientProtocol {
// Set up the ApplicationSubmissionContext
ApplicationSubmissionContext appContext = Records
.newRecord(ApplicationSubmissionContext.class);
-
+
appContext.setApplicationId(applicationId); // ApplicationId
appContext.setResource(capability); // resource
appContext.setQueue( // Queue name
@@ -870,12 +888,12 @@ public class YARNRunner implements ClientProtocol {
Configuration tezJobConf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
// This will replace job.xml in the staging dir.
- writeTezConf(jobSubmitDir, fs, tezJobConf);
+ writeTezConf(jobSubmitDir, fs, tezJobConf);
// FIXME set up job resources
Map<String, LocalResource> jobLocalResources =
createJobLocalResources(tezJobConf, jobSubmitDir);
-
+
// FIXME createDAG should take the tezConf as a parameter, instead of using
// MR keys.
DAG dag = createDAG(fs, jobId, jobConf, jobSubmitDir, ts,
@@ -898,36 +916,6 @@ public class YARNRunner implements ClientProtocol {
|| appMasterReport.getYarnApplicationState() == YarnApplicationState.KILLED) {
throw new IOException("Failed to run job : " + diagnostics);
}
-
- if (LOG.isDebugEnabled()) {
- while (true) {
- appMasterReport = resMgrDelegate.getApplicationReport(applicationId);
- diagnostics = (appMasterReport == null ? "application report is null"
- : appMasterReport.getDiagnostics());
- if (appMasterReport == null) {
- throw new IOException("Failed to run job : " + diagnostics);
- }
- YarnApplicationState state = appMasterReport
- .getYarnApplicationState();
- if (state.equals(YarnApplicationState.FAILED)
- || state.equals(YarnApplicationState.FINISHED)
- || state.equals(YarnApplicationState.KILLED)) {
- LOG.info("Job completed" + ", finalStatus="
- + appMasterReport.getFinalApplicationStatus() + ", finalState="
- + appMasterReport.getYarnApplicationState() + ", diagnostics="
- + diagnostics);
- break;
- } else {
- LOG.info("Job in progress" + ", finalStatus="
- + appMasterReport.getFinalApplicationStatus() + ", finalState="
- + appMasterReport.getYarnApplicationState());
- }
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
- }
- }
} catch (YarnRemoteException e) {
throw new IOException(e);
}
@@ -1089,7 +1077,7 @@ public class YARNRunner implements ClientProtocol {
envConf + " config settings.");
}
}
-
+
@SuppressWarnings("deprecation")
private String getMapJavaOpts(Configuration jobConf) {
// follows pattern from YARN MapReduceChildJVM.java
@@ -1097,9 +1085,9 @@ public class YARNRunner implements ClientProtocol {
adminOpts = jobConf.get(
MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,
MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
-
+
String userOpts = "";
- userOpts =
+ userOpts =
jobConf.get(
MRJobConfig.MAP_JAVA_OPTS, // same as JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS
jobConf.get(
@@ -1109,23 +1097,23 @@ public class YARNRunner implements ClientProtocol {
return adminOpts.trim() + " " + userOpts.trim() + " "
+ getLog4jCmdLineProperties(jobConf, true);
}
-
+
@SuppressWarnings("deprecation")
private String getReduceJavaOpts(Configuration jobConf) {
- // follows pattern from YARN MapReduceChildJVM.java
+ // follows pattern from YARN MapReduceChildJVM.java
String adminOpts = "";
adminOpts = jobConf.get(
MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,
MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
-
+
String userOpts = "";
- userOpts =
+ userOpts =
jobConf.get(
- MRJobConfig.REDUCE_JAVA_OPTS, // same as JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS
+ MRJobConfig.REDUCE_JAVA_OPTS, // same as JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS
jobConf.get(
JobConf.MAPRED_TASK_JAVA_OPTS,
JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
-
+
return adminOpts.trim() + " " + userOpts.trim() + " "
+ getLog4jCmdLineProperties(jobConf, false);
}
@@ -1144,7 +1132,7 @@ public class YARNRunner implements ClientProtocol {
/**
* Add the JVM system properties necessary to configure
* {@link ContainerLogAppender}.
- *
+ *
* @param logLevel
* the desired log level (eg INFO/WARN/DEBUG)
* @param vargs
@@ -1163,4 +1151,4 @@ public class YARNRunner implements ClientProtocol {
vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
}
-}
\ No newline at end of file
+}
[48/50] [abbrv] git commit: TEZ-177. Change Tez to use renamed YARN
exceptions post YARN-635. (sseth)
Posted by ss...@apache.org.
TEZ-177. Change Tez to use renamed YARN exceptions post YARN-635.
(sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/114b12a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/114b12a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/114b12a9
Branch: refs/heads/master
Commit: 114b12a96f1d86fb3c3f01b458dd5b0a69133db6
Parents: 9feab05
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jun 3 22:04:12 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jun 3 22:04:12 2013 -0700
----------------------------------------------------------------------
.../main/java/org/apache/tez/client/TezClient.java | 4 +-
.../hadoop/mapred/MRVertexOutputCommitter.java | 4 +-
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 6 ++--
.../tez/dag/app/TaskAttemptListenerImpTezDag.java | 4 +-
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 4 +-
.../dag/app/launcher/ContainerLauncherImpl.java | 4 +-
.../org/apache/tez/dag/app/rm/TaskScheduler.java | 12 ++++----
.../tez/dag/app/rm/TaskSchedulerEventHandler.java | 6 ++--
.../dag/app/rm/container/AMContainerHelpers.java | 4 +-
.../org/apache/tez/dag/app/rm/node/AMNodeMap.java | 4 +-
.../tez/dag/app/speculate/DefaultSpeculator.java | 10 +++---
.../tez/dag/app/taskclean/TaskCleanerImpl.java | 4 +-
.../apache/tez/engine/runtime/RuntimeUtils.java | 4 +-
.../apache/tez/mapreduce/MiniMRRTezCluster.java | 4 +-
.../tez/mapreduce/ClientServiceDelegate.java | 6 ++--
.../apache/tez/mapreduce/ResourceMgrDelegate.java | 22 +++++++-------
.../java/org/apache/tez/mapreduce/YARNRunner.java | 14 +++++-----
17 files changed, 58 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
index 73c97b3..ec1dc57 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.client.YarnClient;
import org.apache.hadoop.yarn.client.YarnClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
@@ -52,7 +52,7 @@ public class TezClient {
String host = appReport.getHost();
int port = appReport.getRpcPort();
return getDAGClient(host, port);
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new TezException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
index 171c244..b4949c0 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.client.VertexStatus;
import org.apache.tez.dag.api.committer.VertexContext;
@@ -80,7 +80,7 @@ public class MRVertexOutputCommitter extends VertexOutputCommitter {
.getOutputFormatClass(), conf);
committer = outputFormat.getOutputCommitter(taskContext);
} catch (Exception e) {
- throw new YarnException(e);
+ throw new YarnRuntimeException(e);
}
} else {
committer = ReflectionUtils.newInstance(conf.getClass(
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index a4ca97e..96f62a8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -48,7 +48,7 @@ import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.ClusterInfo;
import org.apache.hadoop.yarn.SystemClock;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -579,7 +579,7 @@ public class DAGAppMaster extends CompositeService {
}
}
} catch (IOException e) {
- throw new YarnException(e);
+ throw new YarnRuntimeException(e);
}
}
@@ -827,7 +827,7 @@ public class DAGAppMaster extends CompositeService {
@Override
public Map<ApplicationAccessType, String> getApplicationACLs() {
if (getServiceState() != STATE.STARTED) {
- throw new YarnException(
+ throw new YarnRuntimeException(
"Cannot get ApplicationACLs before all services have started");
}
return taskSchedulerEventHandler.getApplicationAcls();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 708b33e..14e3144 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authorize.PolicyProvider;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.tez.common.ContainerContext;
@@ -129,7 +129,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
server.start();
this.address = NetUtils.getConnectAddress(server);
} catch (IOException e) {
- throw new YarnException(e);
+ throw new YarnRuntimeException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 94e5161..0b6b23a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -583,7 +583,7 @@ public class TaskAttemptImpl implements TaskAttempt,
case SUCCEEDED:
return TaskAttemptState.SUCCEEDED;
default:
- throw new YarnException("Attempt to convert invalid "
+ throw new YarnRuntimeException("Attempt to convert invalid "
+ "stateMachineTaskAttemptState to externalTaskAttemptState: "
+ smState);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index b71af02..5271bae 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
@@ -426,7 +426,7 @@ public class ContainerLauncherImpl extends AbstractService implements
try {
eventQueue.put(event);
} catch (InterruptedException e) {
- throw new YarnException(e);
+ throw new YarnRuntimeException(e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
index d081d2a..58795d7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
@@ -30,7 +30,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.client.AMRMClient.StoredContainerRequest;
import org.apache.hadoop.yarn.client.AMRMClientAsync;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.tez.dag.api.TezException;
@@ -196,12 +196,12 @@ public class TaskScheduler extends AbstractService
response.getMinimumResourceCapability(),
response.getMaximumResourceCapability(),
response.getApplicationACLs());
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
LOG.error("Yarn Exception while registering", e);
- throw new YarnException(e);
+ throw new YarnRuntimeException(e);
} catch (IOException e) {
LOG.error("IO Exception while registering", e);
- throw new YarnException(e);
+ throw new YarnRuntimeException(e);
}
}
@@ -223,7 +223,7 @@ public class TaskScheduler extends AbstractService
// to get our lock.
amRmClient.stop();
super.stop();
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
LOG.error("Yarn Exception while unregistering ", e);
throw new TezException(e);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index e29db1c..7d55bec 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -27,7 +27,7 @@ import java.util.concurrent.LinkedBlockingQueue;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -119,7 +119,7 @@ public class TaskSchedulerEventHandler extends AbstractService
handleTASucceeded(event);
break;
default:
- throw new YarnException("Unexecpted TA_ENDED state: " + event.getState());
+ throw new YarnRuntimeException("Unexecpted TA_ENDED state: " + event.getState());
}
break;
case S_CONTAINER_DEALLOCATE:
@@ -152,7 +152,7 @@ public class TaskSchedulerEventHandler extends AbstractService
try {
eventQueue.put(event);
} catch (InterruptedException e) {
- throw new YarnException(e);
+ throw new YarnRuntimeException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 6156e33..9abff17 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -129,7 +129,7 @@ public class AMContainerHelpers {
ShuffleHandler.serializeServiceData(jobToken));
} catch (IOException e) {
- throw new YarnException(e);
+ throw new YarnRuntimeException(e);
}
// Construct the actual Container
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
index a9256aa..efbdb6b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
@@ -26,7 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -79,7 +79,7 @@ public class AMNodeMap extends AbstractService implements
", maxTaskFailuresPerNode: " + maxTaskFailuresPerNode);
if (blacklistDisablePercent < -1 || blacklistDisablePercent > 100) {
- throw new YarnException("Invalid blacklistDisablePercent: "
+ throw new YarnRuntimeException("Invalid blacklistDisablePercent: "
+ blacklistDisablePercent
+ ". Should be an integer between 0 and 100 or -1 to disabled");
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
index 79ccb8e..c5bb936 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
@@ -35,7 +35,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
@@ -127,16 +127,16 @@ public class DefaultSpeculator extends AbstractService implements
estimator.contextualize(conf, context);
} catch (InstantiationException ex) {
LOG.error("Can't make a speculation runtime extimator", ex);
- throw new YarnException(ex);
+ throw new YarnRuntimeException(ex);
} catch (IllegalAccessException ex) {
LOG.error("Can't make a speculation runtime extimator", ex);
- throw new YarnException(ex);
+ throw new YarnRuntimeException(ex);
} catch (InvocationTargetException ex) {
LOG.error("Can't make a speculation runtime extimator", ex);
- throw new YarnException(ex);
+ throw new YarnRuntimeException(ex);
} catch (NoSuchMethodException ex) {
LOG.error("Can't make a speculation runtime extimator", ex);
- throw new YarnException(ex);
+ throw new YarnRuntimeException(ex);
}
return estimator;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanerImpl.java
index 4e6f897..1590cda 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanerImpl.java
@@ -27,7 +27,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.tez.dag.app.AppContext;
@@ -114,7 +114,7 @@ public class TaskCleanerImpl extends AbstractService implements TaskCleaner {
try {
eventQueue.put(event);
} catch (InterruptedException e) {
- throw new YarnException(e);
+ throw new YarnRuntimeException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
index 1946a62..0dbda40 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
@@ -24,7 +24,7 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.tez.common.InputSpec;
import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.TezEngineTaskContext;
@@ -119,7 +119,7 @@ public class RuntimeUtils {
}
t = createRuntime(taskContext, processor, inputs, outputs);
} catch (ClassNotFoundException e) {
- throw new YarnException("Unable to initialize RuntimeTask, context="
+ throw new YarnRuntimeException("Unable to initialize RuntimeTask, context="
+ taskContext, e);
}
return t;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
index 41a1cf3..bbf5e68 100644
--- a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
+++ b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.util.JarFinder;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
@@ -100,7 +100,7 @@ public class MiniMRRTezCluster extends MiniYARNCluster {
Path doneDirPath = fc.makeQualified(new Path(doneDir));
fc.mkdir(doneDirPath, null, true);
} catch (IOException e) {
- throw new YarnException("Could not create staging directory. ", e);
+ throw new YarnRuntimeException("Could not create staging directory. ", e);
}
conf.set(MRConfig.MASTER_ADDRESS, "test");
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
index 1335a44..88fafbf 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.mapreduce.TaskCompletionEvent;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.client.TezClient;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezRemoteException;
@@ -155,7 +155,7 @@ public class ClientServiceDelegate {
public LogParams getLogFilePath(JobID oldJobID,
TaskAttemptID oldTaskAttemptID)
- throws YarnRemoteException, IOException {
+ throws YarnException, IOException {
// FIXME logs for an attempt?
throw new UnsupportedOperationException();
}
@@ -165,7 +165,7 @@ public class ClientServiceDelegate {
TypeConverter.toYarn(oldJobID);
try {
return rm.getApplicationReport(jobId.getAppId());
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
index 2561b56..8563a09 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.YarnClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ProtoUtils;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
@@ -67,7 +67,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
InterruptedException {
try {
return TypeConverter.fromYarnNodes(super.getNodeReports());
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
}
@@ -75,7 +75,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
public JobStatus[] getAllJobs() throws IOException, InterruptedException {
try {
return TypeConverter.fromYarnApps(super.getApplicationList(), this.conf);
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
}
@@ -92,7 +92,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
YarnClusterMetrics metrics;
try {
metrics = super.getYarnClusterMetrics();
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1,
@@ -107,7 +107,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
try {
return ProtoUtils.convertFromProtoFormat(
super.getRMDelegationToken(renewer), rmAddress);
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
}
@@ -119,7 +119,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
public JobID getNewJobID() throws IOException, InterruptedException {
try {
this.application = super.getNewApplication();
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
this.applicationId = this.application.getApplicationId();
@@ -131,7 +131,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
try {
return TypeConverter.fromYarn(
super.getQueueInfo(queueName), this.conf);
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
}
@@ -141,7 +141,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
try {
return TypeConverter.fromYarnQueueUserAclsInfo(super
.getQueueAclsInfo());
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
}
@@ -149,7 +149,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
public QueueInfo[] getQueues() throws IOException, InterruptedException {
try {
return TypeConverter.fromYarnQueueInfo(super.getAllQueues(), this.conf);
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
}
@@ -157,7 +157,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
public QueueInfo[] getRootQueues() throws IOException, InterruptedException {
try {
return TypeConverter.fromYarnQueueInfo(super.getRootQueueInfos(), this.conf);
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
}
@@ -167,7 +167,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
try {
return TypeConverter.fromYarnQueueInfo(super.getChildQueueInfos(parent),
this.conf);
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/114b12a9/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 05b5411..592648e 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -72,7 +72,7 @@ import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.YarnRuntimeException;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
@@ -87,7 +87,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
@@ -888,7 +888,7 @@ public class YARNRunner implements ClientProtocol {
try {
ts.writeTokenStorageFile(applicationTokensFile, conf);
} catch (IOException e) {
- throw new YarnException(e);
+ throw new YarnRuntimeException(e);
}
FileSystem fs = FileSystem.get(conf);
@@ -924,7 +924,7 @@ public class YARNRunner implements ClientProtocol {
|| appMasterReport.getYarnApplicationState() == YarnApplicationState.KILLED) {
throw new IOException("Failed to run job : " + diagnostics);
}
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
@@ -1007,7 +1007,7 @@ public class YARNRunner implements ClientProtocol {
if (status.getState() != JobStatus.State.RUNNING) {
try {
resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
return;
@@ -1035,7 +1035,7 @@ public class YARNRunner implements ClientProtocol {
if (status.getState() != JobStatus.State.KILLED) {
try {
resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
}
@@ -1070,7 +1070,7 @@ public class YARNRunner implements ClientProtocol {
throws IOException {
try {
return clientCache.getClient(jobID).getLogFilePath(jobID, taskAttemptID);
- } catch (YarnRemoteException e) {
+ } catch (YarnException e) {
throw new IOException(e);
}
}
[35/50] [abbrv] git commit: TEZ-164. Fix handling of completed,
running, pending tasks in AMContainer. (sseth)
Posted by ss...@apache.org.
TEZ-164. Fix handling of completed, running, pending tasks in
AMContainer. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/9f0d4c3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/9f0d4c3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/9f0d4c3c
Branch: refs/heads/master
Commit: 9f0d4c3cb4a8b4b3b6ea89e2c0cb0dc85bc91545
Parents: a74b436
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed May 29 16:35:40 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed May 29 16:35:40 2013 -0700
----------------------------------------------------------------------
.../tez/dag/app/rm/container/AMContainer.java | 3 +-
.../tez/dag/app/rm/container/AMContainerImpl.java | 100 +++++++--------
.../tez/dag/app/rm/container/TestAMContainer.java | 93 ++++++-------
3 files changed, 93 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f0d4c3c/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
index 3c00620..e62d033 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
@@ -30,8 +30,7 @@ public interface AMContainer extends EventHandler<AMContainerEvent>{
public AMContainerState getState();
public ContainerId getContainerId();
public Container getContainer();
- //TODO Rename - CompletedTaskAttempts, ideally means FAILED / KILLED as well.
- public List<TezTaskAttemptID> getCompletedTaskAttempts();
+ public List<TezTaskAttemptID> getAllTaskAttempts();
public TezTaskAttemptID getRunningTaskAttempt();
public List<TezTaskAttemptID> getQueuedTaskAttempts();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f0d4c3c/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 51246dd..58e9d50 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -18,7 +18,6 @@
package org.apache.tez.dag.app.rm.container;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
@@ -48,13 +47,13 @@ import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.dag.event.DiagnosableEvent;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminating;
-import org.apache.tez.dag.app.dag.event.TaskAttemptEventDiagnosticsUpdate;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventNodeFailed;
import org.apache.tez.dag.app.rm.AMSchedulerEventContainerCompleted;
import org.apache.tez.dag.app.rm.AMSchedulerEventDeallocateContainer;
import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
import org.apache.tez.dag.records.TezTaskAttemptID;
+//import org.apache.tez.dag.app.dag.event.TaskAttemptEventDiagnosticsUpdate;
@SuppressWarnings("rawtypes")
public class AMContainerImpl implements AMContainer {
@@ -222,10 +221,19 @@ public class AMContainerImpl implements AMContainer {
}
@Override
- public List<TezTaskAttemptID> getCompletedTaskAttempts() {
+ public List<TezTaskAttemptID> getAllTaskAttempts() {
readLock.lock();
try {
- return new ArrayList<TezTaskAttemptID>(this.completedAttempts);
+ List<TezTaskAttemptID> allAttempts = new LinkedList<TezTaskAttemptID>();
+ allAttempts.addAll(this.completedAttempts);
+ allAttempts.addAll(this.failedAssignments);
+ if (this.pendingAttempt != null) {
+ allAttempts.add(this.pendingAttempt);
+ }
+ if (this.runningAttempt != null) {
+ allAttempts.add(this.runningAttempt);
+ }
+ return allAttempts;
} finally {
readLock.unlock();
}
@@ -359,6 +367,7 @@ public class AMContainerImpl implements AMContainer {
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
AMContainerEventAssignTA event = (AMContainerEventAssignTA) cEvent;
container.inError = true;
+ container.registerFailedAttempt(event.getTaskAttemptId());
container.maybeSendNodeFailureForFailedAssignment(event
.getTaskAttemptId());
container.sendTerminatedToTaskAttempt(event.getTaskAttemptId(),
@@ -378,7 +387,6 @@ public class AMContainerImpl implements AMContainer {
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
AMContainerEventCompleted event = (AMContainerEventCompleted)cEvent;
container.sendCompletedToScheduler();
- container.sendDiagUpdateOnContainerComplete(event);
String diag = event.getContainerStatus().getDiagnostics();
if (!(diag == null || diag.equals(""))) {
LOG.info("Container " + container.getContainerId()
@@ -402,10 +410,11 @@ public class AMContainerImpl implements AMContainer {
}
}
- protected static class NodeFailedAtAllocatedTransition implements
- SingleArcTransition<AMContainerImpl, AMContainerEvent> {
+ protected static class NodeFailedAtAllocatedTransition extends
+ NodeFailedBaseTransition {
@Override
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
+ super.transition(container, cEvent);
container.sendCompletedToScheduler();
container.deAllocate();
}
@@ -414,6 +423,7 @@ public class AMContainerImpl implements AMContainer {
protected static class ErrorTransition extends ErrorBaseTransition {
@Override
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
+ super.transition(container, cEvent);
container.sendCompletedToScheduler();
container.deAllocate();
LOG.info(
@@ -464,7 +474,7 @@ public class AMContainerImpl implements AMContainer {
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
if (container.pendingAttempt != null) {
AMContainerEventLaunchFailed event = (AMContainerEventLaunchFailed) cEvent;
- container.sendTerminatingToTA(container.pendingAttempt,
+ container.sendTerminatingToTaskAttempt(container.pendingAttempt,
event.getMessage());
}
container.unregisterFromTAListener();
@@ -481,9 +491,10 @@ public class AMContainerImpl implements AMContainer {
String errorMessage = getMessage(container, event);
container.sendTerminatedToTaskAttempt(container.pendingAttempt,
errorMessage);
+ container.registerFailedAttempt(container.pendingAttempt);
+ container.pendingAttempt = null;
LOG.warn(errorMessage);
}
- container.sendDiagUpdateOnContainerComplete(event);
container.unregisterFromTAListener();
container.sendCompletedToScheduler();
String diag = event.getContainerStatus().getDiagnostics();
@@ -507,7 +518,7 @@ public class AMContainerImpl implements AMContainer {
@Override
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
if (container.pendingAttempt != null) {
- container.sendTerminatingToTA(container.pendingAttempt,
+ container.sendTerminatingToTaskAttempt(container.pendingAttempt,
getMessage(container, cEvent));
}
container.unregisterFromTAListener();
@@ -541,12 +552,14 @@ public class AMContainerImpl implements AMContainer {
}
if (container.pendingAttempt != null) {
+ // Will be null in COMPLETED state.
container.sendNodeFailureToTA(container.pendingAttempt, errorMessage);
- container.sendTerminatingToTA(container.pendingAttempt, "Node failure");
+ container.sendTerminatingToTaskAttempt(container.pendingAttempt, "Node failure");
}
if (container.runningAttempt != null) {
+ // Will be null in COMPLETED state.
container.sendNodeFailureToTA(container.runningAttempt, errorMessage);
- container.sendTerminatingToTA(container.runningAttempt, "Node failure");
+ container.sendTerminatingToTaskAttempt(container.runningAttempt, "Node failure");
}
}
}
@@ -562,16 +575,17 @@ public class AMContainerImpl implements AMContainer {
}
protected static class ErrorAtLaunchingTransition
- extends ErrorTransition {
+ extends ErrorBaseTransition {
@Override
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
super.transition(container, cEvent);
if (container.pendingAttempt != null) {
- container.sendTerminatedToTaskAttempt(container.pendingAttempt,
+ container.sendTerminatingToTaskAttempt(container.pendingAttempt,
"Container " + container.getContainerId() +
" hit an invalid transition - " + cEvent.getType() + " at " +
container.getState());
}
+ container.sendStopRequestToNM();
container.unregisterFromTAListener();
}
}
@@ -625,7 +639,6 @@ public class AMContainerImpl implements AMContainer {
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
super.transition(container, cEvent);
container.unregisterFromContainerListener();
- container.sendDiagUpdateOnContainerComplete((AMContainerEventCompleted)cEvent);
}
@Override
@@ -707,6 +720,8 @@ public class AMContainerImpl implements AMContainer {
container.sendTerminatedToTaskAttempt(container.runningAttempt,
getMessage(container, event));
container.unregisterAttemptFromListener(container.runningAttempt);
+ container.registerFailedAttempt(container.runningAttempt);
+ container.runningAttempt = null;
super.transition(container, cEvent);
}
}
@@ -716,7 +731,7 @@ public class AMContainerImpl implements AMContainer {
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
container.unregisterAttemptFromListener(container.runningAttempt);
- container.sendTerminatingToTA(container.runningAttempt,
+ container.sendTerminatingToTaskAttempt(container.runningAttempt,
" Container" + container.getContainerId() +
" received a STOP_REQUEST");
super.transition(container, cEvent);
@@ -749,7 +764,7 @@ public class AMContainerImpl implements AMContainer {
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
super.transition(container, cEvent);
container.unregisterAttemptFromListener(container.runningAttempt);
- container.sendTerminatedToTaskAttempt(container.runningAttempt,
+ container.sendTerminatingToTaskAttempt(container.runningAttempt,
"Container " + container.getContainerId() +
" hit an invalid transition - " + cEvent.getType() + " at " +
container.getState());
@@ -766,8 +781,8 @@ public class AMContainerImpl implements AMContainer {
" cannot be allocated to container: " + container.getContainerId() +
" in " + container.getState() + " state";
container.maybeSendNodeFailureForFailedAssignment(event.getTaskAttemptId());
- container.sendTerminatingToTA(event.getTaskAttemptId(), errorMessage);
- container.registerFailedTAAssignment(event.getTaskAttemptId());
+ container.sendTerminatingToTaskAttempt(event.getTaskAttemptId(), errorMessage);
+ container.registerFailedAttempt(event.getTaskAttemptId());
}
}
@@ -788,14 +803,18 @@ public class AMContainerImpl implements AMContainer {
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
AMContainerEventCompleted event = (AMContainerEventCompleted) cEvent;
String diag = event.getContainerStatus().getDiagnostics();
+ for (TezTaskAttemptID taId : container.failedAssignments) {
+ container.sendTerminatedToTaskAttempt(taId, diag);
+ }
if (container.pendingAttempt != null) {
container.sendTerminatedToTaskAttempt(container.pendingAttempt, diag);
+ container.registerFailedAttempt(container.pendingAttempt);
+ container.pendingAttempt = null;
}
if (container.runningAttempt != null) {
container.sendTerminatedToTaskAttempt(container.runningAttempt, diag);
- }
- for (TezTaskAttemptID taId : container.failedAssignments) {
- container.sendTerminatedToTaskAttempt(taId, diag);
+ container.registerFailedAttempt(container.runningAttempt);
+ container.runningAttempt = null;
}
if (!(diag == null || diag.equals(""))) {
LOG.info("Container " + container.getContainerId()
@@ -822,10 +841,9 @@ public class AMContainerImpl implements AMContainer {
}
protected static class ErrorAtNMStopRequestedTransition
- extends ErrorAtStoppingTransition {
+ extends ErrorBaseTransition {
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
super.transition(container, cEvent);
- container.deAllocate();
}
}
@@ -833,15 +851,6 @@ public class AMContainerImpl implements AMContainer {
extends ErrorBaseTransition {
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
super.transition(container, cEvent);
- if (container.pendingAttempt != null) {
- container.sendTerminatedToTaskAttempt(container.pendingAttempt, null);
- }
- if (container.runningAttempt != null) {
- container.sendTerminatedToTaskAttempt(container.runningAttempt, null);
- }
- for (TezTaskAttemptID taId : container.failedAssignments) {
- container.sendTerminatedToTaskAttempt(taId, null);
- }
container.sendCompletedToScheduler();
}
}
@@ -873,7 +882,7 @@ public class AMContainerImpl implements AMContainer {
container.maybeSendNodeFailureForFailedAssignment(event.getTaskAttemptId());
container.sendTerminatedToTaskAttempt(event.getTaskAttemptId(),
errorMessage);
- container.registerFailedTAAssignment(event.getTaskAttemptId());
+ container.registerFailedAttempt(event.getTaskAttemptId());
}
}
@@ -886,17 +895,16 @@ public class AMContainerImpl implements AMContainer {
". Attempts: " + currentTaId + ", " + event.getTaskAttemptId() +
". Current state: " + this.getState();
this.maybeSendNodeFailureForFailedAssignment(event.getTaskAttemptId());
- this.sendTerminatingToTA(event.getTaskAttemptId(), errorMessage);
- this.sendTerminatingToTA(currentTaId, errorMessage);
- this.registerFailedTAAssignment(event.getTaskAttemptId());
+ this.sendTerminatingToTaskAttempt(event.getTaskAttemptId(), errorMessage);
+ this.sendTerminatingToTaskAttempt(currentTaId, errorMessage);
+ this.registerFailedAttempt(event.getTaskAttemptId());
LOG.warn(errorMessage);
this.sendStopRequestToNM();
this.unregisterFromTAListener();
this.unregisterFromContainerListener();
}
-
- protected void registerFailedTAAssignment(TezTaskAttemptID taId) {
+ protected void registerFailedAttempt(TezTaskAttemptID taId) {
failedAssignments.add(taId);
}
@@ -908,23 +916,13 @@ public class AMContainerImpl implements AMContainer {
sendEvent(new AMSchedulerEventContainerCompleted(containerId));
}
- protected void sendDiagUpdateOnContainerComplete(
- AMContainerEventCompleted cEvent) {
- String diag = cEvent.getContainerStatus().getDiagnostics();
- if (pendingAttempt != null) {
- sendEvent(new TaskAttemptEventDiagnosticsUpdate(pendingAttempt, diag));
- }
- if (runningAttempt != null) {
- sendEvent(new TaskAttemptEventDiagnosticsUpdate(runningAttempt, diag));
- }
- }
-
protected void sendTerminatedToTaskAttempt(
TezTaskAttemptID taId, String message) {
sendEvent(new TaskAttemptEventContainerTerminated(taId, message));
}
- protected void sendTerminatingToTA(TezTaskAttemptID taId, String message) {
+ protected void sendTerminatingToTaskAttempt(TezTaskAttemptID taId,
+ String message) {
sendEvent(new TaskAttemptEventContainerTerminating(taId, message));
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f0d4c3c/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index 574a2c5..4541a2e 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -67,7 +67,6 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.junit.Ignore;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
@@ -113,7 +112,7 @@ public class TestAMContainer {
.getTaskAttemptId());
assertEquals(wc.taskAttemptID, wc.amContainer.getRunningTaskAttempt());
assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
-
+
wc.taskAttemptSucceeded(wc.taskAttemptID);
wc.verifyState(AMContainerState.IDLE);
wc.verifyNoOutgoingEvents();
@@ -127,7 +126,7 @@ public class TestAMContainer {
verify(wc.tal).unregisterRunningContainer(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
- assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+ assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
assertFalse(wc.amContainer.isInErrorState());
}
@@ -183,7 +182,7 @@ public class TestAMContainer {
verify(wc.tal).unregisterRunningContainer(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
- assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+ assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
assertFalse(wc.amContainer.isInErrorState());
}
@@ -219,7 +218,7 @@ public class TestAMContainer {
assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
assertNull(wc.amContainer.getRunningTaskAttempt());
- assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+ assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
assertFalse(wc.amContainer.isInErrorState());
}
@@ -258,7 +257,7 @@ public class TestAMContainer {
assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
assertNull(wc.amContainer.getRunningTaskAttempt());
- assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+ assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
assertFalse(wc.amContainer.isInErrorState());
}
@@ -297,8 +296,8 @@ public class TestAMContainer {
AMSchedulerEventType.S_CONTAINER_COMPLETED);
assertNull(wc.amContainer.getRunningTaskAttempt());
-// assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
-// assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+ assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
}
@SuppressWarnings("rawtypes")
@@ -306,16 +305,16 @@ public class TestAMContainer {
public void testAllocationAtRunning() {
WrappedContainer wc = new WrappedContainer();
List<Event> outgoingEvents;
-
+
wc.launchContainer();
wc.containerLaunched();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.pullTaskToRun();
wc.verifyState(AMContainerState.RUNNING);
-
+
TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
wc.assignTaskAttempt(taID2);
-
+
wc.verifyState(AMContainerState.STOP_REQUESTED);
verify(wc.tal).unregisterRunningContainer(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
@@ -326,7 +325,7 @@ public class TestAMContainer {
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
TaskAttemptEventType.TA_CONTAINER_TERMINATING);
assertTrue(wc.amContainer.isInErrorState());
-
+
wc.nmStopSent();
wc.containerCompleted();
// 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
@@ -335,10 +334,10 @@ public class TestAMContainer {
TaskAttemptEventType.TA_CONTAINER_TERMINATED,
TaskAttemptEventType.TA_CONTAINER_TERMINATED,
AMSchedulerEventType.S_CONTAINER_COMPLETED);
-
-// assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
-// assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
-// assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+ assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
}
@SuppressWarnings("rawtypes")
@@ -346,15 +345,15 @@ public class TestAMContainer {
public void testMultipleAllocationsAtLaunching() {
WrappedContainer wc = new WrappedContainer();
List<Event> outgoingEvents;
-
+
wc.launchContainer();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.pullTaskToRun();
wc.verifyState(AMContainerState.LAUNCHING);
-
+
TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
wc.assignTaskAttempt(taID2);
-
+
wc.verifyState(AMContainerState.STOP_REQUESTED);
verify(wc.tal).unregisterRunningContainer(wc.containerID);
verify(wc.chh).unregister(wc.containerID);
@@ -365,7 +364,7 @@ public class TestAMContainer {
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
TaskAttemptEventType.TA_CONTAINER_TERMINATING);
assertTrue(wc.amContainer.isInErrorState());
-
+
wc.nmStopSent();
wc.containerCompleted();
// 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
@@ -374,10 +373,10 @@ public class TestAMContainer {
TaskAttemptEventType.TA_CONTAINER_TERMINATED,
TaskAttemptEventType.TA_CONTAINER_TERMINATED,
AMSchedulerEventType.S_CONTAINER_COMPLETED);
-
-// assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
-// assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
-// assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+ assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
}
@SuppressWarnings("rawtypes")
@@ -385,13 +384,13 @@ public class TestAMContainer {
public void testContainerTimedOutAtRunning() {
WrappedContainer wc = new WrappedContainer();
List<Event> outgoingEvents;
-
+
wc.launchContainer();
wc.containerLaunched();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.pullTaskToRun();
wc.verifyState(AMContainerState.RUNNING);
-
+
wc.containerTimedOut();
wc.verifyState(AMContainerState.STOP_REQUESTED);
verify(wc.tal).unregisterRunningContainer(wc.containerID);
@@ -402,18 +401,18 @@ public class TestAMContainer {
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
// TODO Should this be an RM DE-ALLOCATE instead ?
-
+
wc.containerCompleted();
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
TaskAttemptEventType.TA_CONTAINER_TERMINATED,
AMSchedulerEventType.S_CONTAINER_COMPLETED);
-
+
assertFalse(wc.amContainer.isInErrorState());
-
-// assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
-// assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
-// assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+ assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
}
@SuppressWarnings("rawtypes")
@@ -461,7 +460,6 @@ public class TestAMContainer {
assertFalse(wc.amContainer.isInErrorState());
}
- @Ignore
@SuppressWarnings("rawtypes")
@Test
// Verify that incoming NM launched events to COMPLETED containers are
@@ -484,7 +482,6 @@ public class TestAMContainer {
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
AMSchedulerEventType.S_CONTAINER_COMPLETED,
TaskAttemptEventType.TA_CONTAINER_TERMINATED);
- // TODO Failing because of an extra diagnostic event.
assertFalse(wc.amContainer.isInErrorState());
@@ -495,7 +492,6 @@ public class TestAMContainer {
assertFalse(wc.amContainer.isInErrorState());
}
- @Ignore
@SuppressWarnings("rawtypes")
@Test
public void testContainerCompletedAtIdle() {
@@ -519,7 +515,6 @@ public class TestAMContainer {
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
AMSchedulerEventType.S_CONTAINER_COMPLETED,
TaskAttemptEventType.TA_CONTAINER_TERMINATED);
- // TODO Failing because of two extra diagnostic event.
assertFalse(wc.amContainer.isInErrorState());
@@ -531,8 +526,7 @@ public class TestAMContainer {
assertFalse(wc.amContainer.isInErrorState());
}
-
- @Ignore
+
@SuppressWarnings("rawtypes")
@Test
public void testContainerCompletedAtRunning() {
@@ -557,7 +551,6 @@ public class TestAMContainer {
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
AMSchedulerEventType.S_CONTAINER_COMPLETED,
TaskAttemptEventType.TA_CONTAINER_TERMINATED);
- // TODO Failing because of two extra diagnostic event.
assertFalse(wc.amContainer.isInErrorState());
@@ -693,7 +686,7 @@ public class TestAMContainer {
assertNull(wc.amContainer.getRunningTaskAttempt());
assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
- assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size());
+ assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
}
@SuppressWarnings("rawtypes")
@@ -701,18 +694,18 @@ public class TestAMContainer {
public void testNodeFailedAtRunningMultipleAttempts() {
WrappedContainer wc = new WrappedContainer();
List<Event> outgoingEvents;
-
+
wc.launchContainer();
wc.containerLaunched();
wc.assignTaskAttempt(wc.taskAttemptID);
wc.pullTaskToRun();
wc.taskAttemptSucceeded(wc.taskAttemptID);
-
+
TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
wc.assignTaskAttempt(taID2);
wc.pullTaskToRun();
wc.verifyState(AMContainerState.RUNNING);
-
+
wc.nodeFailed();
// Expecting a complete event from the RM
wc.verifyState(AMContainerState.STOPPING);
@@ -722,14 +715,14 @@ public class TestAMContainer {
TaskAttemptEventType.TA_NODE_FAILED,
TaskAttemptEventType.TA_CONTAINER_TERMINATING,
AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
-
+
for (Event event : outgoingEvents) {
if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
assertEquals("nodeFailed", nfEvent.getDiagnosticInfo());
}
}
-
+
wc.containerCompleted();
outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
verifyUnOrderedOutgoingEventTypes(outgoingEvents,
@@ -737,11 +730,11 @@ public class TestAMContainer {
AMSchedulerEventType.S_CONTAINER_COMPLETED);
assertFalse(wc.amContainer.isInErrorState());
-// assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO. Set/Unset properly.
-// assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
-// assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+ assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
}
-
+
@SuppressWarnings("rawtypes")
@Test
public void testNodeFailedAtCompletedMultipleSuccessfulTAs() {
@@ -771,7 +764,7 @@ public class TestAMContainer {
assertNull(wc.amContainer.getRunningTaskAttempt());
assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
- assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size());
+ assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
}
@SuppressWarnings("rawtypes")
[39/50] [abbrv] git commit: TEZ-173. Fix ContainerLauncher to use
containerToken in the startContainer call [YARN-684]. (sseth)
Posted by ss...@apache.org.
TEZ-173. Fix ContainerLauncher to use containerToken in the
startContainer call [YARN-684]. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a6791898
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a6791898
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a6791898
Branch: refs/heads/master
Commit: a6791898ee72ac1e23aca93b458000e2207bfb35
Parents: 5d0b8ad
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu May 30 22:24:14 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu May 30 22:24:14 2013 -0700
----------------------------------------------------------------------
.../dag/app/launcher/ContainerLauncherImpl.java | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a6791898/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index f1a16b8..5b23c75 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -160,7 +160,7 @@ public class ContainerLauncherImpl extends AbstractService implements
// Now launch the actual container
StartContainerRequest startRequest = Records
.newRecord(StartContainerRequest.class);
- startRequest.setContainer(event.getContainer());
+ startRequest.setContainerToken(event.getContainerToken());
startRequest.setContainerLaunchContext(containerLaunchContext);
StartContainerResponse response = proxy.startContainer(startRequest);
[11/50] [abbrv] git commit: TEZ-31: implement DAG.verify() to
validate structural properties of dag created through tez-dag-api
Posted by ss...@apache.org.
TEZ-31: implement DAG.verify() to validate structural properties of dag created through tez-dag-api
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/645c470e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/645c470e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/645c470e
Branch: refs/heads/master
Commit: 645c470eeb1c4476437498fed639c3c20f57c672
Parents: d660948
Author: mikelid <mi...@microsoft.com>
Authored: Tue May 14 17:13:14 2013 -0700
Committer: mikelid <mi...@microsoft.com>
Committed: Tue May 14 17:13:14 2013 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/tez/dag/api/DAG.java | 149 +++++++++++-
.../java/org/apache/tez/dag/api/TestDAGVerify.java | 194 +++++++++++++++
2 files changed, 340 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/645c470e/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 171921d..300ef31 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Stack;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -87,13 +88,155 @@ public class DAG { // FIXME rename to Topology
this.name = name;
}
- public void verify() throws TezException { // FIXME better exception
- //FIXME are task resources compulsory or will the DAG AM put in a default
- //for each vertex if not specified?
+ // AnnotatedVertex is used by verify()
+ private class AnnotatedVertex {
+ Vertex v;
+
+ int index; //for Tarjan's algorithm
+ int lowlink; //for Tarjan's algorithm
+ boolean onstack; //for Tarjan's algorithm
+
+ int inDegree;
+ int outDegree;
+
+ private AnnotatedVertex(Vertex v){
+ this.v = v;
+ index = -1;
+ lowlink = -1;
+ inDegree = 0;
+ outDegree = 0;
+ }
}
+
+ // verify()
+ //
+ // Default rules
+ // Illegal:
+ // - duplicate vertex id
+ // - cycles
+ //
+ // Ok:
+ // - orphaned vertex. Occurs in map-only
+ // - islands. Occurs if job has unrelated workflows.
+ //
+ // Not yet categorized:
+ // - orphaned vertex in DAG of >1 vertex. Could be unrelated map-only job.
+ // - v1->v2 via two edges. perhaps some self-join job would use this?
+ //
+ // "restricted" mode:
+ // In short term, the supported DAGs are limited. Call with restricted=true for these verifications.
+ // Illegal:
+ // - any vertex with more than one input or output edge. (n-ary input, n-ary merge)
+ public void verify() throws IllegalStateException {
+ verify(true);
+ }
+
+ public void verify(boolean restricted) throws IllegalStateException {
+ Map<Vertex, List<Edge>> edgeMap = new HashMap<Vertex, List<Edge>>();
+ for(Edge e : edges){
+ Vertex inputVertex = e.getInputVertex();
+ List<Edge> edgeList = edgeMap.get(inputVertex);
+ if(edgeList == null){
+ edgeList = new ArrayList<Edge>();
+ edgeMap.put(inputVertex, edgeList);
+ }
+ edgeList.add(e);
+ }
+ // check for duplicate vertex names, and prepare for cycle detection
+ Map<String, AnnotatedVertex> vertexMap = new HashMap<String, AnnotatedVertex>();
+ for(Vertex v : vertices){
+ if(vertexMap.containsKey(v.getVertexName())){
+ throw new IllegalStateException("DAG contains multiple vertices with name: " + v.getVertexName());
+ }
+ vertexMap.put(v.getVertexName(), new AnnotatedVertex(v));
+ }
+
+ detectCycles(edgeMap, vertexMap);
+
+ if(restricted){
+ for(Edge e : edges){
+ vertexMap.get(e.getInputVertex().getVertexName()).outDegree++;
+ vertexMap.get(e.getOutputVertex().getVertexName()).inDegree++;
+ }
+ for(AnnotatedVertex av: vertexMap.values()){
+ if(av.inDegree > 1){
+ throw new IllegalStateException("Vertex has inDegree>1: " + av.v.getVertexName());
+ }
+ if(av.outDegree > 1){
+ throw new IllegalStateException("Vertex has outDegree>1: " + av.v.getVertexName());
+ }
+ }
+ }
+ }
+
+ // Adaptation of Tarjan's algorithm for connected components.
+ // http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
+ private void detectCycles(Map<Vertex, List<Edge>> edgeMap, Map<String, AnnotatedVertex> vertexMap)
+ throws IllegalStateException{
+ Integer nextIndex = 0; // boxed integer so it is passed by reference.
+ Stack<AnnotatedVertex> stack = new Stack<DAG.AnnotatedVertex>();
+ for(AnnotatedVertex av: vertexMap.values()){
+ if(av.index == -1){
+ assert stack.empty();
+ strongConnect(av, vertexMap, edgeMap, stack, nextIndex);
+ }
+ }
+ }
+
+ // part of Tarjan's algorithm for connected components.
+ // http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
+ private void strongConnect(
+ AnnotatedVertex av,
+ Map<String, AnnotatedVertex> vertexMap,
+ Map<Vertex, List<Edge>> edgeMap,
+ Stack<AnnotatedVertex> stack, Integer nextIndex) throws IllegalStateException{
+ av.index = nextIndex;
+ av.lowlink = nextIndex;
+ nextIndex++;
+ stack.push(av);
+ av.onstack = true;
+
+ List<Edge> edges = edgeMap.get(av.v);
+ if(edges != null){
+ for(Edge e : edgeMap.get(av.v)){
+ AnnotatedVertex outVertex = vertexMap.get(e.getOutputVertex().getVertexName());
+ if(outVertex.index == -1){
+ strongConnect(outVertex, vertexMap, edgeMap, stack, nextIndex);
+ av.lowlink = Math.min(av.lowlink, outVertex.lowlink);
+ }
+ else if(outVertex.onstack){
+ // strongly connected component detected, but we will wait till later so that the full cycle can be displayed.
+ // update lowlink in case outputVertex should be considered the root of this component.
+ av.lowlink = Math.min(av.lowlink, outVertex.index);
+ }
+ }
+ }
+
+ if(av.lowlink == av.index ){
+ AnnotatedVertex pop = stack.pop();
+ pop.onstack = false;
+ if(pop != av){
+ // there was something on the stack other than this "av".
+ // this indicates there is a scc/cycle. It comprises all nodes from top of stack to "av"
+ StringBuilder message = new StringBuilder();
+ message.append(av.v.getVertexName() + " <- ");
+ for( ; pop != av; pop = stack.pop()){
+ message.append(pop.v.getVertexName() + " <- ");
+ pop.onstack = false;
+ }
+ message.append(av.v.getVertexName());
+ throw new IllegalStateException("DAG contains a cycle: " + message);
+ }
+ }
+ }
+
+
// create protobuf message describing DAG
public DAGPlan createDag(){
+
+ verify(true);
+
DAGPlan.Builder jobBuilder = DAGPlan.newBuilder();
jobBuilder.setName(this.name);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/645c470e/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java
new file mode 100644
index 0000000..4022604
--- /dev/null
+++ b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
+import org.apache.tez.dag.api.EdgeProperty.SourceType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestDAGVerify {
+
+ private final String dummyProcessorClassName = TestDAGVerify.class.getName();
+ private final String dummyInputClassName = TestDAGVerify.class.getName();
+ private final String dummyOutputClassName = TestDAGVerify.class.getName();
+ private final int dummyTaskCount = 2;
+
+ // v1
+ // |
+ // v2
+ @Test
+ public void testVerify1() {
+ Vertex v1 = new Vertex("v1",dummyProcessorClassName, dummyTaskCount);
+ Vertex v2 = new Vertex("v2","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Edge e1 = new Edge(v1, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, dummyInputClassName, dummyOutputClassName));
+ DAG dag = new DAG();
+ dag.addVertex(v1);
+ dag.addVertex(v2);
+ dag.addEdge(e1);
+ dag.verify();
+ }
+
+ // v1 <----
+ // | ^
+ // v2 ^
+ // | | ^
+ // v3 v4
+ @Test
+ public void testCycle1() {
+ IllegalStateException ex=null;
+ Vertex v1 = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Vertex v2 = new Vertex("v2","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Vertex v3 = new Vertex("v3","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Vertex v4 = new Vertex("v4","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Edge e1 = new Edge(v1, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ Edge e2 = new Edge(v2, v3, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ Edge e3 = new Edge(v2, v4, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ Edge e4 = new Edge(v4, v1, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ DAG dag = new DAG();
+ dag.addVertex(v1);
+ dag.addVertex(v2);
+ dag.addVertex(v3);
+ dag.addVertex(v4);
+ dag.addEdge(e1);
+ dag.addEdge(e2);
+ dag.addEdge(e3);
+ dag.addEdge(e4);
+ try{
+ dag.verify();
+ }
+ catch (IllegalStateException e){
+ ex = e;
+ }
+ Assert.assertNotNull(ex);
+ System.out.println(ex.getMessage());
+ Assert.assertTrue(ex.getMessage().startsWith("DAG contains a cycle"));
+ }
+
+// v1
+// |
+// -> v2
+// ^ | |
+// v3 v4
+ @Test
+ public void testCycle2() {
+ IllegalStateException ex=null;
+ Vertex v1 = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Vertex v2 = new Vertex("v2","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Vertex v3 = new Vertex("v3","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Vertex v4 = new Vertex("v4","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Edge e1 = new Edge(v1, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ Edge e2 = new Edge(v2, v3, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ Edge e3 = new Edge(v2, v4, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ Edge e4 = new Edge(v3, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ DAG dag = new DAG();
+ dag.addVertex(v1);
+ dag.addVertex(v2);
+ dag.addVertex(v3);
+ dag.addVertex(v4);
+ dag.addEdge(e1);
+ dag.addEdge(e2);
+ dag.addEdge(e3);
+ dag.addEdge(e4);
+ try{
+ dag.verify();
+ }
+ catch (IllegalStateException e){
+ ex = e;
+ }
+ Assert.assertNotNull(ex);
+ System.out.println(ex.getMessage());
+ Assert.assertTrue(ex.getMessage().startsWith("DAG contains a cycle"));
+ }
+
+ @Test
+ public void repeatedVertexName() {
+ IllegalStateException ex=null;
+ Vertex v1 = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Vertex v1repeat = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ DAG dag = new DAG();
+ dag.addVertex(v1);
+ dag.addVertex(v1repeat);
+ try {
+ dag.verify();
+ }
+ catch (IllegalStateException e){
+ ex = e;
+ }
+ Assert.assertNotNull(ex);
+ System.out.println(ex.getMessage());
+ Assert.assertTrue(ex.getMessage().startsWith("DAG contains multiple vertices with name"));
+ }
+
+ // v1 v2
+ // | |
+ // v3
+ @Test
+ public void BinaryInput() {
+ IllegalStateException ex=null;
+ try {
+ Vertex v1 = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Vertex v2 = new Vertex("v2","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Vertex v3 = new Vertex("v3","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Edge e1 = new Edge(v1, v3, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ Edge e2 = new Edge(v2, v3, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ DAG dag = new DAG();
+ dag.addVertex(v1);
+ dag.addVertex(v2);
+ dag.addVertex(v3);
+ dag.addEdge(e1);
+ dag.addEdge(e2);
+ dag.verify();
+ }
+ catch (IllegalStateException e){
+ ex = e;
+ }
+ Assert.assertNotNull(ex);
+ System.out.println(ex.getMessage());
+ Assert.assertTrue(ex.getMessage().startsWith("Vertex has inDegree>1"));
+ }
+
+ // v1
+ // | |
+ // v2 v3
+ @Test
+ public void BinaryOutput() {
+ IllegalStateException ex=null;
+ try {
+ Vertex v1 = new Vertex("v1","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Vertex v2 = new Vertex("v2","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Vertex v3 = new Vertex("v3","org.apache.tez.mapreduce.processor.reduce.MapProcessor", dummyTaskCount);
+ Edge e1 = new Edge(v1, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ Edge e2 = new Edge(v1, v2, new EdgeProperty(ConnectionPattern.BIPARTITE, SourceType.STABLE, "dummy input class", "dummy output class"));
+ DAG dag = new DAG();
+ dag.addVertex(v1);
+ dag.addVertex(v2);
+ dag.addVertex(v3);
+ dag.addEdge(e1);
+ dag.addEdge(e2);
+ dag.verify();
+ }
+ catch (IllegalStateException e){
+ ex = e;
+ }
+ Assert.assertNotNull(ex);
+ System.out.println(ex.getMessage());
+ Assert.assertTrue(ex.getMessage().startsWith("Vertex has outDegree>1"));
+ }
+}
[45/50] [abbrv] git commit: TEZ-178. Change YARN object construction
to be based on newInstance in various classes instead of BuilderUtils.
(sseth)
Posted by ss...@apache.org.
TEZ-178. Change YARN object construction to be based on newInstance in
various classes instead of BuilderUtils. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/9f040cf8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/9f040cf8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/9f040cf8
Branch: refs/heads/master
Commit: 9f040cf8b33600233be2bd2b9795406816c687e4
Parents: 8652329
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jun 3 13:35:56 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jun 3 13:35:56 2013 -0700
----------------------------------------------------------------------
.../main/java/org/apache/tez/common/IDUtils.java | 5 +-
.../java/org/apache/tez/dag/records/TezDAGID.java | 5 +-
.../org/apache/tez/records/TezContainerId.java | 11 ++-
.../org/apache/tez/dag/api/DagTypeConverters.java | 5 +-
.../org/apache/tez/dag/api/oldrecords/AMInfo.java | 2 +-
.../org/apache/tez/dag/app/MRRExampleHelper.java | 10 +-
.../apache/tez/dag/app/TaskHeartbeatHandler.java | 2 +-
.../tez/dag/app/dag/TaskAttemptStateInternal.java | 2 +-
.../TaskAttemptEventContainerTerminating.java | 2 +-
.../app/dag/event/TaskAttemptEventFailRequest.java | 2 +-
.../app/dag/event/TaskAttemptEventNodeFailed.java | 2 +-
.../tez/dag/app/dag/impl/DAGSchedulerMRR.java | 8 +-
.../dag/app/dag/impl/DAGSchedulerNaturalOrder.java | 8 +-
.../tez/dag/app/rm/AMSchedulerEventTAEnded.java | 2 +-
.../app/rm/AMSchedulerEventTALaunchRequest.java | 2 +-
.../rm/container/AMContainerEventNodeFailed.java | 2 +-
.../dag/app/rm/container/AMContainerHelpers.java | 3 +-
.../tez/dag/app/rm/container/AMContainerMap.java | 2 +-
.../app/rm/node/AMNodeEventTaskAttemptEnded.java | 2 +-
.../tez/dag/app/rm/node/AMNodeEventType.java | 2 +-
.../org/apache/tez/dag/app/rm/node/AMNodeMap.java | 2 +-
.../org/apache/tez/dag/utils/TezBuilderUtils.java | 2 +-
.../apache/tez/dag/app/dag/impl/TestDAGImpl.java | 6 +-
.../tez/dag/app/dag/impl/TestTaskAttempt.java | 74 +++++++-------
.../apache/tez/dag/app/dag/impl/TestTaskImpl.java | 5 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 6 +-
.../apache/tez/dag/app/rm/TestTaskScheduler.java | 6 +-
.../tez/dag/app/rm/container/TestAMContainer.java | 15 ++--
.../apache/tez/mapreduce/hadoop/IDConverter.java | 3 +-
.../hadoop/MultiStageMRConfToTezTranslator.java | 2 +-
.../org/apache/tez/mapreduce/TezTestUtils.java | 10 +-
.../org/apache/tez/mapreduce/NotRunningJob.java | 1 -
.../java/org/apache/tez/mapreduce/YARNRunner.java | 7 +-
33 files changed, 107 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/IDUtils.java b/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
index e2cbf35..e94d939 100644
--- a/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
+++ b/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
@@ -19,7 +19,6 @@
package org.apache.tez.common;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezID;
import org.apache.tez.dag.records.TezTaskAttemptID;
@@ -41,7 +40,7 @@ public class IDUtils {
String[] parts = str.split("_");
if(parts.length == 6) {
if(parts[0].equals(TezTaskID.TASK)) {
- ApplicationId appId = BuilderUtils.newApplicationId(
+ ApplicationId appId = ApplicationId.newInstance(
Long.valueOf(parts[1]), Integer.parseInt(parts[2]));
TezDAGID dagId = new TezDAGID(appId, Integer.parseInt(parts[3]));
TezVertexID vId = new TezVertexID(dagId, Integer.parseInt(parts[4]));
@@ -71,7 +70,7 @@ public class IDUtils {
String[] parts = str.split(Character.toString(TezID.SEPARATOR));
if(parts.length == 7) {
if(parts[0].equals(TezTaskAttemptID.ATTEMPT)) {
- ApplicationId appId = BuilderUtils.newApplicationId(
+ ApplicationId appId = ApplicationId.newInstance(
Long.valueOf(parts[1]), Integer.parseInt(parts[2]));
TezDAGID dagId = new TezDAGID(appId, Integer.parseInt(parts[3]));
TezVertexID vId = new TezVertexID(dagId, Integer.parseInt(parts[4]));
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
index 733a908..9a0a409 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.text.NumberFormat;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.BuilderUtils;
/**
@@ -71,7 +70,7 @@ public class TezDAGID extends TezID {
* @param id the dag number
*/
public TezDAGID(String yarnRMIdentifier, int appId, int id) {
- this(BuilderUtils.newApplicationId(Long.valueOf(yarnRMIdentifier),
+ this(ApplicationId.newInstance(Long.valueOf(yarnRMIdentifier),
appId), id);
}
@@ -117,7 +116,7 @@ public class TezDAGID extends TezID {
@Override
public void readFields(DataInput in) throws IOException {
- applicationId = BuilderUtils.newApplicationId(in.readLong(), in.readInt());
+ applicationId = ApplicationId.newInstance(in.readLong(), in.readInt());
super.readFields(in);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java b/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
index 24c1e8a..193b979 100644
--- a/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
+++ b/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
@@ -24,8 +24,9 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.BuilderUtils;
//TODO EVENTUALLY Once everything is on PB, get rid of this.
//Alternately have the PB interfaces implement Writable.
@@ -58,8 +59,12 @@ public class TezContainerId implements Writable {
int appId = in.readInt();
int appAttemptId = in.readInt();
int id = in.readInt();
- this.containerId = BuilderUtils.newContainerId(appId, appAttemptId,
- timestamp, id);
+
+ ApplicationId applicationId = ApplicationId.newInstance(timestamp, appId);
+ ApplicationAttemptId applicationAttemptId = ApplicationAttemptId
+ .newInstance(applicationId, appAttemptId);
+
+ this.containerId = ContainerId.newInstance(applicationAttemptId, id);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index ba42e23..6437080 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeConnectionPattern;
@@ -207,7 +206,7 @@ public class DagTypeConverters {
public static Resource CreateResourceRequestFromTaskConfig(
PlanTaskConfiguration taskConfig) {
- return BuilderUtils.newResource(taskConfig.getMemoryMb(), taskConfig.getVirtualCores());
+ return Resource.newInstance(taskConfig.getMemoryMb(), taskConfig.getVirtualCores());
}
public static Map<String, String> createSettingsMapFromDAGPlan(
@@ -218,4 +217,4 @@ public class DagTypeConverters {
}
return map;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
index 7726d7b..6cb3df0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
@@ -35,4 +35,4 @@ public interface AMInfo {
public void setNodeManagerHost(String nmHost);
public void setNodeManagerPort(int nmPort);
public void setNodeManagerHttpPort(int mnHttpPort);
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
index 92f0e87..35ae558 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
@@ -16,7 +16,6 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeProperty;
@@ -126,12 +125,13 @@ public class MRRExampleHelper {
reduce2Rsrcs.put(MRJobConfig.JOB_CONF_FILE, jobRsrcs.get(MRJobConfig.JOB_CONF_FILE));
reduce2Rsrcs.put(getConfFileName("reduce2"), jobRsrcs.get(getConfFileName("reduce2")));
- Resource mapResource = BuilderUtils.newResource(
+ Resource mapResource = Resource.newInstance(
MRJobConfig.DEFAULT_MAP_MEMORY_MB,
MRJobConfig.DEFAULT_MAP_CPU_VCORES);
+
mapVertex.setTaskResource(mapResource);
mapVertex.setTaskLocalResources(mapRsrcs);
- Resource reduceResource = BuilderUtils.newResource(
+ Resource reduceResource = Resource.newInstance(
MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
MRJobConfig.DEFAULT_REDUCE_CPU_VCORES);
reduce1Vertex.setTaskResource(reduceResource);
@@ -182,12 +182,12 @@ public class MRRExampleHelper {
reduceRsrcs.put(MRJobConfig.JOB_CONF_FILE, jobRsrcs.get(MRJobConfig.JOB_CONF_FILE));
reduceRsrcs.put(getConfFileName("reduce"), jobRsrcs.get(MRJobConfig.JOB_CONF_FILE));
- Resource mapResource = BuilderUtils.newResource(
+ Resource mapResource = Resource.newInstance(
MRJobConfig.DEFAULT_MAP_MEMORY_MB,
MRJobConfig.DEFAULT_MAP_CPU_VCORES);
mapVertex.setTaskResource(mapResource);
mapVertex.setTaskLocalResources(mapRsrcs);
- Resource reduceResource = BuilderUtils.newResource(
+ Resource reduceResource = Resource.newInstance(
MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
MRJobConfig.DEFAULT_REDUCE_CPU_VCORES);
reduceVertex.setTaskResource(reduceResource);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
index 7239484..b6fdc53 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
@@ -65,4 +65,4 @@ public class TaskHeartbeatHandler extends HeartbeatHandlerBase<TezTaskAttemptID>
eventHandler.handle(new TaskAttemptEvent(attemptId,
TaskAttemptEventType.TA_TIMED_OUT));
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
index 3633b08..9ad5460 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
@@ -36,4 +36,4 @@ public enum TaskAttemptStateInternal {
KILLED,
FAILED,
SUCCEEDED
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
index 8c88cb9..7da6e14 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
@@ -35,4 +35,4 @@ public class TaskAttemptEventContainerTerminating extends TaskAttemptEvent
return this.message;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
index bba8027..b2f9cfb 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
@@ -32,4 +32,4 @@ public class TaskAttemptEventFailRequest extends TaskAttemptEvent {
public String getMessage() {
return this.message;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
index 4fc718d..ee143bb 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
@@ -33,4 +33,4 @@ public class TaskAttemptEventNodeFailed extends TaskAttemptEvent {
return this.message;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
index d404155..4f51c02 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
@@ -20,8 +20,8 @@ package org.apache.tez.dag.app.dag.impl;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.DAGScheduler;
@@ -124,9 +124,9 @@ public class DAGSchedulerMRR implements DAGScheduler {
" with depth " + vertexDistanceFromRoot +
" at priority " + priority);
- TaskAttemptEventSchedule attemptEvent =
- new TaskAttemptEventSchedule(attempt.getID(),
- BuilderUtils.newPriority(priority));
+ TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule(
+ attempt.getID(), Priority.newInstance(priority));
+
sendEvent(attemptEvent);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
index 9e8729e..6fec907 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
@@ -20,8 +20,8 @@ package org.apache.tez.dag.app.dag.impl;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.DAGScheduler;
import org.apache.tez.dag.app.dag.TaskAttempt;
@@ -66,9 +66,9 @@ public class DAGSchedulerNaturalOrder implements DAGScheduler {
LOG.info("Scheduling " + attempt.getID() + " at priority " + priority);
- TaskAttemptEventSchedule attemptEvent =
- new TaskAttemptEventSchedule(attempt.getID(),
- BuilderUtils.newPriority(priority));
+ TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule(
+ attempt.getID(), Priority.newInstance(priority));
+
sendEvent(attemptEvent);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
index b2a90f2..90e76b7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
@@ -51,4 +51,4 @@ public class AMSchedulerEventTAEnded extends AMSchedulerEvent {
public ContainerId getUsedContainerId() {
return this.containerId;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
index b595c2b..313ee7d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
@@ -130,4 +130,4 @@ public class AMSchedulerEventTALaunchRequest extends AMSchedulerEvent {
* - Credentials, tokens etc are identical.
* + Command - dependent on map / reduce java.opts
*/
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java
index c1c603c..9f5d39c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java
@@ -36,4 +36,4 @@ public class AMContainerEventNodeFailed extends AMContainerEvent implements
return message;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index c17f7f2..6156e33 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.app.AppContext;
@@ -76,7 +75,7 @@ public class AMContainerHelpers {
long resourceSize = fstat.getLen();
long resourceModificationTime = fstat.getModificationTime();
- return BuilderUtils.newLocalResource(resourceURL, type, visibility,
+ return LocalResource.newInstance(resourceURL, type, visibility,
resourceSize, resourceModificationTime);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
index 708838c..5f5a7be 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java
@@ -72,4 +72,4 @@ public class AMContainerMap extends AbstractService implements
public Collection<AMContainer> values() {
return containerMap.values();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
index cc2b7f7..c823236 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
@@ -50,4 +50,4 @@ public class AMNodeEventTaskAttemptEnded extends AMNodeEvent {
public TezTaskAttemptID getTaskAttemptId() {
return this.taskAttemptId;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java
index 60534fd..421e482 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java
@@ -38,4 +38,4 @@ public enum AMNodeEventType {
// Producer: AMNode - Will not reach AMNodeImpl. Used to compute whether
// blacklisting should be ignored.
N_NODE_WAS_BLACKLISTED
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
index 206244e..a9256aa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeMap.java
@@ -210,4 +210,4 @@ public class AMNodeMap extends AbstractService implements
public boolean isBlacklistingIgnored() {
return this.ignoreBlacklisting;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
index 60ec0a0..a734bb8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
@@ -51,4 +51,4 @@ public class TezBuilderUtils {
return new TezTaskID(newVertexID(dagId, vertexId), taskId);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index 9e94cbd..9588099 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -27,9 +27,9 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.SystemClock;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
@@ -343,8 +343,8 @@ public class TestDAGImpl {
@Before
public void setup() {
conf = new TezConfiguration();
- appAttemptId = BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(100, 1), 1);
+ appAttemptId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(100, 1), 1);
dagId = new TezDAGID(appAttemptId.getApplicationId(), 1);
Assert.assertNotNull(dagId);
dagPlan = createTestDAGPlan();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index 9601418..516c656 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -49,10 +49,10 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.common.TezTaskContext;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
@@ -131,7 +131,7 @@ public class TestTaskAttempt {
mock(TaskAttemptListener.class), 1, new TezConfiguration(),
mock(Token.class), new Credentials(), new SystemClock(),
mock(TaskHeartbeatHandler.class), mock(AppContext.class),
- MAP_PROCESSOR_NAME, locationHint, BuilderUtils.newResource(1024, 1),
+ MAP_PROCESSOR_NAME, locationHint, Resource.newInstance(1024, 1),
new HashMap<String, LocalResource>(), new HashMap<String, String>(),
"", false);
@@ -175,7 +175,7 @@ public class TestTaskAttempt {
mock(TaskAttemptListener.class), 1, new TezConfiguration(),
mock(Token.class), new Credentials(), new SystemClock(),
mock(TaskHeartbeatHandler.class), mock(AppContext.class),
- MAP_PROCESSOR_NAME, locationHint, BuilderUtils.newResource(1024, 1),
+ MAP_PROCESSOR_NAME, locationHint, Resource.newInstance(1024, 1),
new HashMap<String, LocalResource>(), new HashMap<String, String>(),
"", false);
TaskAttemptImpl spyTa = spy(taImpl);
@@ -222,8 +222,8 @@ public class TestTaskAttempt {
// Configuration conf = new Configuration();
// conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
// conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
- // app.setClusterInfo(new ClusterInfo(BuilderUtils
- // .newResource(minContainerSize, 1), BuilderUtils.newResource(10240,1)));
+ // app.setClusterInfo(new ClusterInfo(Resource.newInstance(minContainerSize, 1),
+ // Resource.newInstance(10240,1)));
//
// Job job = app.submit(conf);
// app.waitForState(job, JobState.RUNNING);
@@ -294,8 +294,8 @@ public class TestTaskAttempt {
// Ensure the dag does not go into an error state if a attempt kill is
// received while STARTING
public void testLaunchFailedWhileKilling() throws Exception {
- ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
- ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ ApplicationId appId = ApplicationId.newInstance(1, 2);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
appId, 0);
TezDAGID dagID = new TezDAGID(appId, 1);
TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -313,7 +313,7 @@ public class TestTaskAttempt {
TaskLocationHint locationHint = new TaskLocationHint(
new String[] { "127.0.0.1" }, null);
- Resource resource = BuilderUtils.newResource(1024, 1);
+ Resource resource = Resource.newInstance(1024, 1);
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
Map<String, String> environment = new HashMap<String, String>();
String javaOpts = "";
@@ -327,14 +327,14 @@ public class TestTaskAttempt {
MAP_PROCESSOR_NAME, locationHint, resource, localResources,
environment, javaOpts, false);
- NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
- ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+ ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
Container container = mock(Container.class);
when(container.getId()).thenReturn(contId);
when(container.getNodeId()).thenReturn(nid);
- taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, BuilderUtils
- .newPriority(3)));
+ taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, Priority
+ .newInstance(3)));
// At state STARTING.
taImpl.handle(new TaskAttemptEventKillRequest(taskAttemptID, null));
// At some KILLING state.
@@ -348,8 +348,8 @@ public class TestTaskAttempt {
// Ensure ContainerTerminating and ContainerTerminated is handled correctly by
// the TaskAttempt
public void testContainerTerminationWhileRunning() throws Exception {
- ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
- ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ ApplicationId appId = ApplicationId.newInstance(1, 2);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
appId, 0);
TezDAGID dagID = new TezDAGID(appId, 1);
TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -367,13 +367,13 @@ public class TestTaskAttempt {
TaskLocationHint locationHint = new TaskLocationHint(
new String[] { "127.0.0.1" }, null);
- Resource resource = BuilderUtils.newResource(1024, 1);
+ Resource resource = Resource.newInstance(1024, 1);
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
Map<String, String> environment = new HashMap<String, String>();
String javaOpts = "";
- NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
- ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+ ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
Container container = mock(Container.class);
when(container.getId()).thenReturn(contId);
when(container.getNodeId()).thenReturn(nid);
@@ -443,8 +443,8 @@ public class TestTaskAttempt {
@Test
// Ensure ContainerTerminated is handled correctly by the TaskAttempt
public void testContainerTerminatedWhileRunning() throws Exception {
- ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
- ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ ApplicationId appId = ApplicationId.newInstance(1, 2);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
appId, 0);
TezDAGID dagID = new TezDAGID(appId, 1);
TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -462,13 +462,13 @@ public class TestTaskAttempt {
TaskLocationHint locationHint = new TaskLocationHint(
new String[] { "127.0.0.1" }, null);
- Resource resource = BuilderUtils.newResource(1024, 1);
+ Resource resource = Resource.newInstance(1024, 1);
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
Map<String, String> environment = new HashMap<String, String>();
String javaOpts = "";
- NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
- ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+ ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
Container container = mock(Container.class);
when(container.getId()).thenReturn(contId);
when(container.getNodeId()).thenReturn(nid);
@@ -508,8 +508,8 @@ public class TestTaskAttempt {
@Test
// Ensure ContainerTerminated is handled correctly by the TaskAttempt
public void testContainerTerminatedWhileCommitting() throws Exception {
- ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
- ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ ApplicationId appId = ApplicationId.newInstance(1, 2);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
appId, 0);
TezDAGID dagID = new TezDAGID(appId, 1);
TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -527,13 +527,13 @@ public class TestTaskAttempt {
TaskLocationHint locationHint = new TaskLocationHint(
new String[] { "127.0.0.1" }, null);
- Resource resource = BuilderUtils.newResource(1024, 1);
+ Resource resource = Resource.newInstance(1024, 1);
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
Map<String, String> environment = new HashMap<String, String>();
String javaOpts = "";
- NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
- ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+ ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
Container container = mock(Container.class);
when(container.getId()).thenReturn(contId);
when(container.getNodeId()).thenReturn(nid);
@@ -575,8 +575,8 @@ public class TestTaskAttempt {
// Ensure ContainerTerminating and ContainerTerminated is handled correctly by
// the TaskAttempt
public void testContainerTerminatedAfterSuccess() throws Exception {
- ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
- ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ ApplicationId appId = ApplicationId.newInstance(1, 2);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
appId, 0);
TezDAGID dagID = new TezDAGID(appId, 1);
TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -594,13 +594,13 @@ public class TestTaskAttempt {
TaskLocationHint locationHint = new TaskLocationHint(
new String[] { "127.0.0.1" }, null);
- Resource resource = BuilderUtils.newResource(1024, 1);
+ Resource resource = Resource.newInstance(1024, 1);
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
Map<String, String> environment = new HashMap<String, String>();
String javaOpts = "";
- NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
- ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+ ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
Container container = mock(Container.class);
when(container.getId()).thenReturn(contId);
when(container.getNodeId()).thenReturn(nid);
@@ -667,8 +667,8 @@ public class TestTaskAttempt {
// Verifies that multiple TooManyFetchFailures are handled correctly by the
// TaskAttempt.
public void testMultipleTooManyFetchFailures() throws Exception {
- ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
- ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ ApplicationId appId = ApplicationId.newInstance(1, 2);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
appId, 0);
TezDAGID dagID = new TezDAGID(appId, 1);
TezVertexID vertexID = new TezVertexID(dagID, 1);
@@ -687,13 +687,13 @@ public class TestTaskAttempt {
TaskLocationHint locationHint = new TaskLocationHint(
new String[] { "127.0.0.1" }, null);
- Resource resource = BuilderUtils.newResource(1024, 1);
+ Resource resource = Resource.newInstance(1024, 1);
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
Map<String, String> environment = new HashMap<String, String>();
String javaOpts = "";
- NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
- ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
+ NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+ ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
Container container = mock(Container.class);
when(container.getId()).thenReturn(contId);
when(container.getNodeId()).thenReturn(nid);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 44e7686..f2ab880 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.event.InlineDispatcher;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
@@ -101,11 +100,11 @@ public class TestTaskImpl {
clock = new SystemClock();
locationHint = new TaskLocationHint(new String[1], new String[1]);
- appId = BuilderUtils.newApplicationId(System.currentTimeMillis(), 1);
+ appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
dagId = new TezDAGID(appId, 1);
vertexId = new TezVertexID(dagId, 1);
appContext = mock(AppContext.class);
- taskResource = BuilderUtils.newResource(1024, 1);
+ taskResource = Resource.newInstance(1024, 1);
localResources = new HashMap<String, LocalResource>();
environment = new HashMap<String, String>();
javaOpts = "";
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 41b500f..e3af5b3 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -34,9 +34,9 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.SystemClock;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
@@ -463,8 +463,8 @@ public class TestVertexImpl {
@Before
public void setup() {
- appAttemptId = BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(100, 1), 1);
+ appAttemptId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(100, 1), 1);
dagId = new TezDAGID(appAttemptId.getApplicationId(), 1);
dagPlan = createTestDAGPlan();
dispatcher = new DrainDispatcher();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
index 8d52e9a..7036e09 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -40,7 +41,6 @@ import org.apache.hadoop.yarn.client.AMRMClientAsync;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.tez.dag.app.rm.TaskScheduler.CookieContainerRequest;
import org.apache.tez.dag.app.rm.TaskScheduler.TaskSchedulerAppCallback;
@@ -68,8 +68,8 @@ public class TestTaskScheduler {
mock(AMRMClientAsync.class);
ApplicationAttemptId attemptId =
- BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(1234, 0), 0);
+ ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(1234, 0), 0);
String appHost = "host";
int appPort = 0;
String appUrl = "url";
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index cf01f43..332f737 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.common.TezTaskContext;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.app.AppContext;
@@ -831,14 +830,14 @@ public class TestAMContainer {
@SuppressWarnings("unchecked")
public WrappedContainer() {
- applicationID = BuilderUtils.newApplicationId(rmIdentifier, 1);
- appAttemptID = BuilderUtils.newApplicationAttemptId(applicationID, 1);
- containerID = BuilderUtils.newContainerId(appAttemptID, 1);
- nodeID = BuilderUtils.newNodeId("host", 12500);
+ applicationID = ApplicationId.newInstance(rmIdentifier, 1);
+ appAttemptID = ApplicationAttemptId.newInstance(applicationID, 1);
+ containerID = ContainerId.newInstance(appAttemptID, 1);
+ nodeID = NodeId.newInstance("host", 12500);
nodeHttpAddress = "host:12501";
- resource = BuilderUtils.newResource(1024, 1);
- priority = BuilderUtils.newPriority(1);
- container = BuilderUtils.newContainer(containerID, nodeID,
+ resource = Resource.newInstance(1024, 1);
+ priority = Priority.newInstance(1);
+ container = Container.newInstance(containerID, nodeID,
nodeHttpAddress, resource, priority, null);
chh = mock(ContainerHeartbeatHandler.class);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
index 4046596..f7c4f50 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapred.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
@@ -55,7 +54,7 @@ public class IDConverter {
// FIXME hardcoded assumption that one app is one dag
public static TezDAGID fromMRJobId(
org.apache.hadoop.mapreduce.JobID jobId) {
- return new TezDAGID(BuilderUtils.newApplicationId(
+ return new TezDAGID(ApplicationId.newInstance(
Long.valueOf(jobId.getJtIdentifier()), jobId.getId()), 1);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
index c444314..9f1ae88 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
@@ -189,4 +189,4 @@ public class MultiStageMRConfToTezTranslator {
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
index 2e823e6..9a3a666 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
@@ -17,7 +17,7 @@
*/
package org.apache.tez.mapreduce;
-import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
@@ -31,7 +31,7 @@ public class TezTestUtils {
new TezTaskID(
new TezVertexID(
new TezDAGID(
- BuilderUtils.newApplicationId(0, jobId), jobId),
+ ApplicationId.newInstance(0, jobId), jobId),
vertexId),
taskId)
, taskAttemptId);
@@ -41,20 +41,20 @@ public class TezTestUtils {
int vertexId, int taskId) {
return new TezTaskID(
new TezVertexID(new TezDAGID(
- BuilderUtils.newApplicationId(0, jobId),
+ ApplicationId.newInstance(0, jobId),
jobId), vertexId),
taskId);
}
public static TezDAGID getMockJobId(int jobId) {
return new TezDAGID(
- BuilderUtils.newApplicationId(0, jobId), jobId);
+ ApplicationId.newInstance(0, jobId), jobId);
}
public static TezVertexID getMockVertexId(int jobId, int vId) {
return new TezVertexID(
new TezDAGID(
- BuilderUtils.newApplicationId(0, jobId), jobId),
+ ApplicationId.newInstance(0, jobId), jobId),
vId);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
index ba7ab5a..c29102b 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/NotRunningJob.java
@@ -68,7 +68,6 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.util.BuilderUtils;
public class NotRunningJob implements MRClientProtocol {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f040cf8/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 572437b..e2ef5b6 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -89,7 +89,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.tez.dag.api.DAG;
@@ -486,7 +485,7 @@ public class YARNRunner implements ClientProtocol {
Map<String, String> reduceEnv = new HashMap<String, String>();
setupMapReduceEnv(conf, reduceEnv, false);
- Resource reduceResource = BuilderUtils.newResource(conf.getInt(
+ Resource reduceResource = Resource.newInstance(conf.getInt(
MRJobConfig.REDUCE_MEMORY_MB, MRJobConfig.DEFAULT_REDUCE_MEMORY_MB),
conf.getInt(MRJobConfig.REDUCE_CPU_VCORES,
MRJobConfig.DEFAULT_REDUCE_CPU_VCORES));
@@ -559,7 +558,7 @@ public class YARNRunner implements ClientProtocol {
TaskLocationHint[] inputSplitLocations =
getMapLocationHintsFromInputSplits(jobId, fs, jobConf, jobSubmitDir);
- Resource mapResource = BuilderUtils.newResource(
+ Resource mapResource = Resource.newInstance(
jobConf.getInt(MRJobConfig.MAP_MEMORY_MB,
MRJobConfig.DEFAULT_MAP_MEMORY_MB),
jobConf.getInt(MRJobConfig.MAP_CPU_VCORES,
@@ -602,7 +601,7 @@ public class YARNRunner implements ClientProtocol {
Map<String, String> reduceEnv = new HashMap<String, String>();
setupMapReduceEnv(jobConf, reduceEnv, false);
- Resource reduceResource = BuilderUtils.newResource(
+ Resource reduceResource = Resource.newInstance(
jobConf.getInt(MRJobConfig.REDUCE_MEMORY_MB,
MRJobConfig.DEFAULT_REDUCE_MEMORY_MB),
jobConf.getInt(MRJobConfig.REDUCE_CPU_VCORES,
[41/50] [abbrv] git commit: TEZ-172. TestVertexImpl hangs
(intermittent failure).
Posted by ss...@apache.org.
TEZ-172. TestVertexImpl hangs (intermittent failure).
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/d94d37af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/d94d37af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/d94d37af
Branch: refs/heads/master
Commit: d94d37afbe9a0cc045e2c8717c93fa5446b589b9
Parents: d3b1921
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri May 31 17:10:14 2013 -0700
Committer: Mike Liddell <ml...@apache.org>
Committed: Fri May 31 18:09:37 2013 -0700
----------------------------------------------------------------------
pom.xml | 3 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 6 +-
.../apache/tez/dag/app/dag/impl/TestDAGImpl.java | 68 +++--
.../tez/dag/app/dag/impl/TestVertexImpl.java | 262 ++++++++++-----
4 files changed, 218 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d94d37af/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 721df2f..4c5d0c5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -325,7 +325,8 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>2.14.1</version>
<configuration>
- <forkMode>always</forkMode>
+ <forkCount>1</forkCount>
+ <reuseForks>false</reuseForks>
<forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
<argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d94d37af/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index c7bceb6..b06e264 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -613,7 +613,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
getStateMachine().doTransition(event.getType(), event);
} catch (InvalidStateTransitonException e) {
String message = "Invalid event " + event.getType() +
- " on vertex " + this.vertexId +
+ " on vertex " + this.vertexName +
+ " with vertexId " + this.vertexId +
" at current state " + oldState;
LOG.error("Can't handle " + message, e);
addDiagnostic(message);
@@ -630,6 +631,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
finally {
writeLock.unlock();
}
+ LOG.info("DEBUG: Finished processing VertexEvent " + event.getVertexId()
+ + " of type " + event.getType() + " while in state "
+ + getInternalState() + ". Event: " + event);
}
private VertexState getInternalState() {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d94d37af/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index 3adcd8b..9e94cbd 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -373,19 +373,23 @@ public class TestDAGImpl {
@After
public void teardown() {
- dagPlan = null;
- dag = null;
dispatcher.await();
dispatcher.stop();
+ dagPlan = null;
+ dag = null;
}
private void initDAG(DAGImpl dag) {
- dag.handle(new DAGEvent(dagId, DAGEventType.DAG_INIT));
+ dag.handle(
+ new DAGEvent(dagId, DAGEventType.DAG_INIT));
Assert.assertEquals(DAGState.INITED, dag.getState());
}
+ @SuppressWarnings("unchecked")
private void startDAG(DAGImpl dag) {
- dag.handle(new DAGEvent(dagId, DAGEventType.DAG_START));
+ dispatcher.getEventHandler().handle(
+ new DAGEvent(dagId, DAGEventType.DAG_START));
+ dispatcher.await();
Assert.assertEquals(DAGState.RUNNING, dag.getState());
}
@@ -432,9 +436,9 @@ public class TestDAGImpl {
TezVertexID vId = new TezVertexID(dagId, 1);
Vertex v = dag.getVertex(vId);
- ((EventHandler<VertexEvent>) v).handle(new VertexEventTaskCompleted(
+ dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(
new TezTaskID(vId, 0), TaskState.SUCCEEDED));
- ((EventHandler<VertexEvent>) v).handle(new VertexEventTaskCompleted(
+ dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(
new TezTaskID(vId, 1), TaskState.SUCCEEDED));
dispatcher.await();
@@ -442,12 +446,14 @@ public class TestDAGImpl {
Assert.assertEquals(1, dag.getSuccessfulVertices());
}
+ @SuppressWarnings("unchecked")
public void testKillStartedDAG() {
initDAG(dag);
startDAG(dag);
dispatcher.await();
- dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+ dispatcher.getEventHandler().handle(
+ new DAGEvent(dagId, DAGEventType.DAG_KILL));
dispatcher.await();
Assert.assertEquals(DAGState.KILLED, dag.getState());
@@ -479,7 +485,7 @@ public class TestDAGImpl {
Assert.assertEquals(VertexState.SUCCEEDED, v0.getState());
Assert.assertEquals(VertexState.RUNNING, v1.getState());
- dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+ dispatcher.getEventHandler().handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
dispatcher.await();
Assert.assertEquals(DAGState.KILL_WAIT, dag.getState());
@@ -493,13 +499,16 @@ public class TestDAGImpl {
Assert.assertEquals(1, dag.getSuccessfulVertices());
}
+ @SuppressWarnings("unchecked")
@Test
public void testInvalidEvent() {
- dag.handle(new DAGEvent(dagId, DAGEventType.DAG_START));
+ dispatcher.getEventHandler().handle(
+ new DAGEvent(dagId, DAGEventType.DAG_START));
dispatcher.await();
Assert.assertEquals(DAGState.ERROR, dag.getState());
}
+ @SuppressWarnings("unchecked")
@Test
@Ignore
public void testVertexSuccessfulCompletionUpdates() {
@@ -508,28 +517,29 @@ public class TestDAGImpl {
dispatcher.await();
for (int i = 0; i < 6; ++i) {
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
}
dispatcher.await();
Assert.assertEquals(DAGState.RUNNING, dag.getState());
Assert.assertEquals(1, dag.getSuccessfulVertices());
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 2), VertexState.SUCCEEDED));
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 3), VertexState.SUCCEEDED));
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 4), VertexState.SUCCEEDED));
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 5), VertexState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(DAGState.SUCCEEDED, dag.getState());
Assert.assertEquals(6, dag.getSuccessfulVertices());
}
+ @SuppressWarnings("unchecked")
@Test
@Ignore
public void testVertexFailureHandling() {
@@ -537,14 +547,14 @@ public class TestDAGImpl {
startDAG(dag);
dispatcher.await();
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(DAGState.RUNNING, dag.getState());
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 2), VertexState.FAILED));
dispatcher.await();
Assert.assertEquals(DAGState.FAILED, dag.getState());
@@ -558,6 +568,7 @@ public class TestDAGImpl {
}
}
+ @SuppressWarnings("unchecked")
@Test
@Ignore
public void testDAGKill() {
@@ -565,17 +576,18 @@ public class TestDAGImpl {
startDAG(dag);
dispatcher.await();
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(DAGState.RUNNING, dag.getState());
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
- dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+ dispatcher.getEventHandler().handle(
+ new DAGEvent(dagId, DAGEventType.DAG_KILL));
for (int i = 2; i < 6; ++i) {
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, i), VertexState.SUCCEEDED));
}
dispatcher.await();
@@ -584,29 +596,31 @@ public class TestDAGImpl {
Assert.assertEquals(1, dagFinishEventHandler.dagFinishEvents);
}
+ @SuppressWarnings("unchecked")
@Test
public void testDAGKillPending() {
initDAG(dag);
startDAG(dag);
dispatcher.await();
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(DAGState.RUNNING, dag.getState());
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
- dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+ dispatcher.getEventHandler().handle(
+ new DAGEvent(dagId, DAGEventType.DAG_KILL));
for (int i = 2; i < 5; ++i) {
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, i), VertexState.SUCCEEDED));
}
dispatcher.await();
Assert.assertEquals(DAGState.KILL_WAIT, dag.getState());
- dag.handle(new DAGEventVertexCompleted(
+ dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(
new TezVertexID(dagId, 5), VertexState.KILLED));
dispatcher.await();
Assert.assertEquals(DAGState.KILLED, dag.getState());
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d94d37af/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index e4a1040..41b500f 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -413,11 +413,12 @@ public class TestVertexImpl {
vertexIdMap = new HashMap<TezVertexID, VertexImpl>();
for (int i = 0; i < vCnt; ++i) {
VertexPlan vPlan = dagPlan.getVertex(i);
+ String vName = vPlan.getName();
TezVertexID vertexId = new TezVertexID(dagId, i+1);
VertexImpl v = new VertexImpl(vertexId, vPlan, vPlan.getName(), conf,
dispatcher.getEventHandler(), taskAttemptListener, jobToken, fsTokens,
clock, thh, appContext, vertexLocationHint);
- vertices.put(vPlan.getName(), v);
+ vertices.put(vName, v);
vertexIdMap.put(vertexId, v);
}
}
@@ -492,16 +493,29 @@ public class TestVertexImpl {
@After
public void teardown() {
+ dispatcher.await();
+ dispatcher.stop();
+ dispatcher = null;
+ vertexEventDispatcher = null;
+ dagEventDispatcher = null;
dagPlan = null;
this.vertices = null;
this.edges = null;
- dispatcher.await();
- dispatcher.stop();
+ this.vertexIdMap = null;
}
+ private void initAllVertices() {
+ for (int i = 1; i <= 6; ++i) {
+ VertexImpl v = vertices.get("vertex" + i);
+ initVertex(v);
+ }
+ }
+
+
+ @SuppressWarnings("unchecked")
private void initVertex(VertexImpl v) {
Assert.assertEquals(VertexState.NEW, v.getState());
- v.handle(new VertexEvent(v.getVertexId(),
+ dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(),
VertexEventType.V_INIT));
dispatcher.await();
Assert.assertEquals(VertexState.INITED, v.getState());
@@ -511,8 +525,10 @@ public class TestVertexImpl {
startVertex(v, true);
}
+ @SuppressWarnings("unchecked")
private void killVertex(VertexImpl v, boolean checkKillWait) {
- v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+ dispatcher.getEventHandler().handle(
+ new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
dispatcher.await();
if (checkKillWait) {
Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
@@ -521,10 +537,11 @@ public class TestVertexImpl {
}
}
+ @SuppressWarnings("unchecked")
private void startVertex(VertexImpl v,
boolean checkRunningState) {
Assert.assertEquals(VertexState.INITED, v.getState());
- v.handle(new VertexEvent(v.getVertexId(),
+ dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(),
VertexEventType.V_START));
dispatcher.await();
if (checkRunningState) {
@@ -532,7 +549,7 @@ public class TestVertexImpl {
}
}
- @Test
+ @Test(timeout = 5000)
public void testVertexInit() {
VertexImpl v = vertices.get("vertex2");
initVertex(v);
@@ -583,66 +600,79 @@ public class TestVertexImpl {
.getOutputClassName()));
}
- @Test
+ @Test(timeout = 5000)
public void testVertexStart() {
+ initAllVertices();
+
VertexImpl v = vertices.get("vertex2");
- initVertex(v);
startVertex(v);
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
public void testBasicVertexCompletion() {
+ initAllVertices();
+
VertexImpl v = vertices.get("vertex2");
- initVertex(v);
startVertex(v);
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
- v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, v.getState());
Assert.assertEquals(1, v.getCompletedTasks());
- v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(2, v.getCompletedTasks());
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
@Ignore // FIXME fix verteximpl for this test to work
public void testDuplicateTaskCompletion() {
+ initAllVertices();
+
VertexImpl v = vertices.get("vertex2");
- initVertex(v);
startVertex(v);
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
- v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, v.getState());
- v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, v.getState());
- v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
public void testVertexFailure() {
+ initAllVertices();
+
VertexImpl v = vertices.get("vertex2");
- initVertex(v);
startVertex(v);
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
- v.handle(new VertexEventTaskCompleted(t1, TaskState.FAILED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.FAILED));
dispatcher.await();
Assert.assertEquals(VertexState.FAILED, v.getState());
String diagnostics =
@@ -650,17 +680,18 @@ public class TestVertexImpl {
Assert.assertTrue(diagnostics.contains("task failed " + t1.toString()));
}
- @Test
+ @Test(timeout = 5000)
public void testVertexWithNoTasks() {
// FIXME a vertex with no tasks should not be allowed
+ initAllVertices();
+
VertexImpl v = vertices.get("vertex1");
- initVertex(v);
startVertex(v, false);
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
}
- @Test
+ @Test(timeout = 5000)
public void testVertexKillDiagnostics() {
VertexImpl v1 = vertices.get("vertex1");
killVertex(v1, false);
@@ -679,7 +710,14 @@ public class TestVertexImpl {
"vertex received kill in inited state"));
VertexImpl v3 = vertices.get("vertex3");
+ VertexImpl v4 = vertices.get("vertex4");
+ VertexImpl v5 = vertices.get("vertex5");
+ VertexImpl v6 = vertices.get("vertex6");
initVertex(v3);
+ initVertex(v4);
+ initVertex(v5);
+ initVertex(v6);
+
startVertex(v3);
killVertex(v3, true);
diagnostics =
@@ -688,68 +726,77 @@ public class TestVertexImpl {
"vertex received kill while in running state"));
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
public void testVertexKillPending() {
- VertexImpl v = vertices.get("vertex2");
- initVertex(v);
- VertexImpl v3 = vertices.get("vertex3");
- initVertex(v3);
+ initAllVertices();
+ VertexImpl v = vertices.get("vertex2");
startVertex(v);
- v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+ dispatcher.getEventHandler().handle(
+ new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+ dispatcher.await();
Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
- v.handle(new VertexEventTaskCompleted(
- new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(
+ new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
+ dispatcher.await();
Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
- v.handle(new VertexEventTaskCompleted(
- new TezTaskID(v.getVertexId(), 1), TaskState.KILLED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(
+ new TezTaskID(v.getVertexId(), 1), TaskState.KILLED));
dispatcher.await();
Assert.assertEquals(VertexState.KILLED, v.getState());
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
@Ignore
public void testVertexKill() {
- VertexImpl v = vertices.get("vertex2");
- initVertex(v);
- VertexImpl v3 = vertices.get("vertex3");
- initVertex(v3);
+ initAllVertices();
+ VertexImpl v = vertices.get("vertex2");
startVertex(v);
- v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+ dispatcher.getEventHandler().handle(
+ new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
- v.handle(new VertexEventTaskCompleted(
- new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(
+ new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
- v.handle(new VertexEventTaskCompleted(
- new TezTaskID(v.getVertexId(), 1), TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(
+ new TezTaskID(v.getVertexId(), 1), TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.KILLED, v.getState());
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
@Ignore
public void testKilledTasksHandling() {
+ initAllVertices();
+
VertexImpl v = vertices.get("vertex2");
- initVertex(v);
startVertex(v);
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
- v.handle(new VertexEventTaskCompleted(t1, TaskState.FAILED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.FAILED));
dispatcher.await();
Assert.assertEquals(VertexState.FAILED, v.getState());
Assert.assertEquals(TaskState.KILLED, v.getTask(t2).getState());
}
- @Test
+ @Test(timeout = 5000)
public void testVertexCommitterInit() {
VertexImpl v2 = vertices.get("vertex2");
initVertex(v2);
@@ -762,7 +809,7 @@ public class TestVertexImpl {
instanceof MRVertexOutputCommitter);
}
- @Test
+ @Test(timeout = 5000)
public void testVertexSchedulerInit() {
VertexImpl v2 = vertices.get("vertex2");
initVertex(v2);
@@ -775,10 +822,13 @@ public class TestVertexImpl {
instanceof BipartiteSlowStartVertexScheduler);
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
public void testVertexTaskFailure() {
+ initAllVertices();
+
VertexImpl v = vertices.get("vertex2");
- initVertex(v);
+
CountingVertexOutputCommitter committer =
new CountingVertexOutputCommitter();
v.setVertexOutputCommitter(committer);
@@ -787,62 +837,66 @@ public class TestVertexImpl {
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
- v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, v.getState());
- v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
- v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t2, TaskState.FAILED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t2, TaskState.FAILED));
dispatcher.await();
Assert.assertEquals(VertexState.FAILED, v.getState());
Assert.assertEquals(0, committer.commitCounter);
Assert.assertEquals(1, committer.abortCounter);
}
- @Test
+ @Test(timeout = 5000)
public void testSourceVertexStartHandling() {
+ LOG.info("Testing testSourceVertexStartHandling");
+ initAllVertices();
+
VertexImpl v4 = vertices.get("vertex4");
- initVertex(v4);
VertexImpl v5 = vertices.get("vertex5");
- initVertex(v5);
VertexImpl v6 = vertices.get("vertex6");
- initVertex(v6);
- Assert.assertEquals(VertexState.INITED, v6.getState());
startVertex(v4);
startVertex(v5);
dispatcher.await();
+ LOG.info("Verifying v6 state " + v6.getState());
Assert.assertEquals(VertexState.RUNNING, v6.getState());
Assert.assertEquals(1, v6.getDistanceFromRoot());
}
- @Test
+ @Test(timeout = 5000)
public void testCounters() {
// FIXME need to test counters at vertex level
}
- @Test
+ @Test(timeout = 5000)
public void testDiagnostics() {
// FIXME need to test diagnostics in various cases
}
- @Test
+ @Test(timeout = 5000)
public void testTaskAttemptCompletionEvents() {
// FIXME need to test handling of task attempt events
}
- @Test
+ @Test(timeout = 5000)
public void testSourceTaskAttemptCompletionEvents() {
+ LOG.info("Testing testSourceTaskAttemptCompletionEvents");
+ initAllVertices();
+
VertexImpl v4 = vertices.get("vertex4");
- initVertex(v4);
VertexImpl v5 = vertices.get("vertex5");
- initVertex(v5);
VertexImpl v6 = vertices.get("vertex6");
- initVertex(v6);
startVertex(v4);
startVertex(v5);
dispatcher.await();
+ LOG.info("Verifying v6 state " + v6.getState());
Assert.assertEquals(VertexState.RUNNING, v6.getState());
TezTaskID t1_v4 = new TezTaskID(v4.getVertexId(), 0);
@@ -897,17 +951,21 @@ public class TestVertexImpl {
Assert.assertEquals(6, v6.getTaskAttemptCompletionEvents(0, 100).length);
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
public void testDAGEventGeneration() {
+ initAllVertices();
+
VertexImpl v = vertices.get("vertex2");
- initVertex(v);
startVertex(v);
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
- v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
- v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(1,
@@ -915,11 +973,14 @@ public class TestVertexImpl {
DAGEventType.DAG_VERTEX_COMPLETED).intValue());
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
public void testTaskReschedule() {
// For downstream failures
+ initAllVertices();
+
VertexImpl v = vertices.get("vertex2");
- initVertex(v);
+
CountingVertexOutputCommitter committer =
new CountingVertexOutputCommitter();
v.setVertexOutputCommitter(committer);
@@ -929,26 +990,33 @@ public class TestVertexImpl {
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
- v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
- v.handle(new VertexEventTaskReschedule(t1));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskReschedule(t1));
// FIXME need to handle dups
- // v.handle(new VertexEventTaskReschedule(t1));
- v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ // dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, v.getState());
Assert.assertEquals(0, committer.commitCounter);
- v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(1, committer.commitCounter);
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
public void testVertexCommit() {
+ initAllVertices();
+
VertexImpl v = vertices.get("vertex2");
- initVertex(v);
+
CountingVertexOutputCommitter committer =
new CountingVertexOutputCommitter();
v.setVertexOutputCommitter(committer);
@@ -958,14 +1026,18 @@ public class TestVertexImpl {
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
- v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
- v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
- v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(1, committer.commitCounter);
- v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(1, committer.commitCounter);
@@ -974,20 +1046,23 @@ public class TestVertexImpl {
Assert.assertEquals(0, committer.setupCounter); // already done in init
}
- @Test
+ @Test(timeout = 5000)
public void testCommitterInitAndSetup() {
// FIXME need to add a test for this
}
- @Test
+ @Test(timeout = 5000)
public void testTaskAttemptFetchFailureHandling() {
// FIXME needs testing
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
public void testBadCommitter() {
+ initAllVertices();
+
VertexImpl v = vertices.get("vertex2");
- initVertex(v);
+
CountingVertexOutputCommitter committer =
new CountingVertexOutputCommitter(true, true);
v.setVertexOutputCommitter(committer);
@@ -997,8 +1072,10 @@ public class TestVertexImpl {
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
- v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
- v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.getEventHandler().handle(
+ new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.FAILED, v.getState());
Assert.assertEquals(1, committer.commitCounter);
@@ -1009,14 +1086,15 @@ public class TestVertexImpl {
Assert.assertEquals(0, committer.setupCounter); // already done in init
}
- @Test
+ @Test(timeout = 5000)
public void testHistoryEventGeneration() {
}
- @Test
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
public void testInvalidEvent() {
VertexImpl v = vertices.get("vertex2");
- v.handle(new VertexEvent(v.getVertexId(),
+ dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(),
VertexEventType.V_START));
dispatcher.await();
Assert.assertEquals(VertexState.ERROR, v.getState());
[49/50] [abbrv] git commit: TEZ-179. Update JobContextImpl to use the
changed return type for certain methods post MAPREDUCE-5300. (sseth)
Posted by ss...@apache.org.
TEZ-179. Update JobContextImpl to use the changed return type for
certain methods post MAPREDUCE-5300. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a10e607a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a10e607a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a10e607a
Branch: refs/heads/master
Commit: a10e607ac5f3eeae71df8dc32c08b95c3a2ebd82
Parents: 114b12a
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jun 3 22:06:03 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jun 3 22:06:03 2013 -0700
----------------------------------------------------------------------
.../mapreduce/hadoop/mapreduce/JobContextImpl.java | 24 +++++++++++++--
1 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a10e607a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
index 1f34fdb..4962260 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
@@ -352,7 +352,25 @@ public class JobContextImpl implements JobContext {
public Path[] getFileClassPaths() {
return DistributedCache.getFileClassPaths(conf);
}
-
+
+ /**
+ * Parse a list of longs into strings.
+ *
+ * @param timestamps
+ * the list of longs to parse
+ * @return a list of string that were parsed. same length as timestamps.
+ */
+ private static String[] toTimestampStrs(long[] timestamps) {
+ if (timestamps == null) {
+ return null;
+ }
+ String[] result = new String[timestamps.length];
+ for (int i = 0; i < timestamps.length; ++i) {
+ result[i] = Long.toString(timestamps[i]);
+ }
+ return result;
+ }
+
/**
* Get the timestamps of the archives. Used by internal
* DistributedCache and MapReduce code.
@@ -360,7 +378,7 @@ public class JobContextImpl implements JobContext {
* @throws IOException
*/
public String[] getArchiveTimestamps() {
- return DistributedCache.getArchiveTimestamps(conf);
+ return toTimestampStrs(DistributedCache.getArchiveTimestamps(conf));
}
/**
@@ -370,7 +388,7 @@ public class JobContextImpl implements JobContext {
* @throws IOException
*/
public String[] getFileTimestamps() {
- return DistributedCache.getFileTimestamps(conf);
+ return toTimestampStrs(DistributedCache.getFileTimestamps(conf));
}
/**
[13/50] [abbrv] git commit: TEZ-62. Fix and enable tests in
TestMapProcessor and TestReduceProcessor. (sseth)
Posted by ss...@apache.org.
TEZ-62. Fix and enable tests in TestMapProcessor and
TestReduceProcessor. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/cee79f63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/cee79f63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/cee79f63
Branch: refs/heads/master
Commit: cee79f6384940ca226cd0c2ef02e024f19e7550a
Parents: 38b410e
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri May 17 00:03:20 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Fri May 17 00:03:20 2013 -0700
----------------------------------------------------------------------
.../java/org/apache/tez/common/TezJobConfig.java | 7 +
.../org/apache/hadoop/mapred/YarnTezDagChild.java | 10 +-
.../apache/tez/common/TezEngineTaskContext.java | 4 +-
.../mapreduce/split/SplitMetaInfoReaderTez.java | 14 ++-
.../tez/mapreduce/hadoop/TezTypeConverters.java | 2 +
.../org/apache/tez/mapreduce/processor/MRTask.java | 8 +-
.../tez/mapreduce/processor/map/MapProcessor.java | 14 ++-
.../tez/mapreduce/TestUmbilicalProtocol.java | 2 +-
.../apache/tez/mapreduce/processor/MapUtils.java | 85 ++++++++----
.../mapreduce/processor/map/TestMapProcessor.java | 79 +++++++++---
.../processor/reduce/TestReduceProcessor.java | 104 +++++++++++----
11 files changed, 236 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
index e867458..4a7abea 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -68,6 +68,13 @@ public class TezJobConfig {
public static final String DEFAULT_LOCAL_DIR = "/tmp";
/**
+ * The directory which contains the localized files for this task.
+ */
+ @Private
+ public static final String TASK_LOCAL_RESOURCE_DIR = "tez.engine.task-local-resource.dir";
+ public static final String DEFAULT_TASK_LOCAL_RESOURCE_DIR = "/tmp";
+
+ /**
*
*/
public static final String TEZ_ENGINE_TASK_INDEGREE =
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index f140c3a..2295b6a 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -249,12 +249,16 @@ public class YarnTezDagChild {
* out an output directory.
* @throws IOException
*/
- private static void configureLocalDirs(MRTask task, JobConf job) throws IOException {
+ private static void configureLocalDirs(JobConf job) throws IOException {
String[] localSysDirs = StringUtils.getTrimmedStrings(
System.getenv(Environment.LOCAL_DIRS.name()));
job.setStrings(TezJobConfig.LOCAL_DIR, localSysDirs);
+ job.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
+ System.getenv(Environment.PWD.name()));
LOG.info(TezJobConfig.LOCAL_DIR + " for child: " +
job.get(TezJobConfig.LOCAL_DIR));
+ LOG.info(TezJobConfig.TASK_LOCAL_RESOURCE_DIR + " for child: "
+ + job.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
LocalDirAllocator lDirAlloc = new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
Path workDir = null;
// First, try to find the JOB_LOCAL_DIR on this host.
@@ -285,7 +289,7 @@ public class YarnTezDagChild {
}
}
// TODO TEZ This likely needs fixing to make sure things work when there are multiple local-dirs etc.
- job.set(MRJobConfig.JOB_LOCAL_DIR,workDir.toString());
+ job.set(MRJobConfig.JOB_LOCAL_DIR, workDir.toString());
}
private static JobConf configureTask(MRTask task, Credentials credentials,
@@ -308,7 +312,7 @@ public class YarnTezDagChild {
// JobTokenSecretManager.createSecretKey(jt.getPassword()));
// setup the child's MRConfig.LOCAL_DIR.
- configureLocalDirs(task, job);
+ configureLocalDirs(job);
// setup the child's attempt directories
// Do the task-type specific localization
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
index 350702a..5823de6 100644
--- a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
+++ b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
@@ -39,7 +39,7 @@ public class TezEngineTaskContext extends TezTaskContext {
}
public TezEngineTaskContext(TezTaskAttemptID taskAttemptID, String user,
- String jobName, String vertexName, String moduleClassName,
+ String jobName, String vertexName, String processorName,
List<InputSpec> inputSpecList, List<OutputSpec> outputSpecList) {
super(taskAttemptID, user, jobName, vertexName);
this.inputSpecList = inputSpecList;
@@ -52,7 +52,7 @@ public class TezEngineTaskContext extends TezTaskContext {
}
this.inputSpecList = inputSpecList;
this.outputSpecList = outputSpecList;
- this.processorName = moduleClassName;
+ this.processorName = processorName;
}
public String getProcessorName() {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
index de8d972..72246c0 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
+import org.apache.tez.common.TezJobConfig;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
/**
@@ -44,6 +45,9 @@ public class SplitMetaInfoReaderTez {
public static final Log LOG = LogFactory.getLog(SplitMetaInfoReaderTez.class);
+ public static final int META_SPLIT_VERSION = JobSplit.META_SPLIT_VERSION;
+ public static final byte[] META_SPLIT_FILE_HEADER = JobSplit.META_SPLIT_FILE_HEADER;
+
// Forked from the MR variant so that the metaInfo file as well as the split
// file can be read from local fs - relying on these files being localized.
public static TaskSplitMetaInfo[] readSplitMetaInfo(Configuration conf,
@@ -53,9 +57,11 @@ public class SplitMetaInfoReaderTez {
MRJobConfig.SPLIT_METAINFO_MAXSIZE,
MRJobConfig.DEFAULT_SPLIT_METAINFO_MAXSIZE);
- Path metaSplitFile = new Path(MRJobConfig.JOB_SPLIT_METAINFO);
+ Path metaSplitFile = new Path(
+ conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR),
+ MRJobConfig.JOB_SPLIT_METAINFO);
String jobSplitFile = MRJobConfig.JOB_SPLIT;
-
+
File file = new File(metaSplitFile.toUri().getPath()).getAbsoluteFile();
LOG.info("DEBUG: Setting up JobSplitIndex with JobSplitFile at: "
+ file.getAbsolutePath() + ", defaultFS from conf: "
@@ -83,12 +89,12 @@ public class SplitMetaInfoReaderTez {
JobSplit.SplitMetaInfo splitMetaInfo = new JobSplit.SplitMetaInfo();
splitMetaInfo.readFields(in);
JobSplit.TaskSplitIndex splitIndex = new JobSplit.TaskSplitIndex(
- jobSplitFile, splitMetaInfo.getStartOffset());
+ new Path(conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR), jobSplitFile)
+ .toUri().toString(), splitMetaInfo.getStartOffset());
allSplitMetaInfo[i] = new JobSplit.TaskSplitMetaInfo(splitIndex,
splitMetaInfo.getLocations(), splitMetaInfo.getInputDataLength());
}
in.close();
return allSplitMetaInfo;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
index 1fc71b0..768d347 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
@@ -34,6 +34,8 @@ import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
public class TezTypeConverters {
+ // TODO Remove unused methods
+
// Tez objects will be imported. Others will use the fully qualified name when
// required.
// All public methods named toYarn / toTez / toMapReduce
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
index d7d721f..276294c 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
@@ -149,7 +149,7 @@ extends RunningTaskContext {
jobConf.set(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID,
taskAttemptId.toString());
-
+
initResourceCalculatorPlugin();
LOG.info("MRTask.inited: taskAttemptId = " + taskAttemptId.toString());
@@ -695,8 +695,10 @@ extends RunningTaskContext {
public void localizeConfiguration(JobConf jobConf)
throws IOException, InterruptedException {
- jobConf.set(JobContext.TASK_ID, taskAttemptId.getTaskID().toString());
- jobConf.set(JobContext.TASK_ATTEMPT_ID, taskAttemptId.toString());
+ jobConf.set(JobContext.TASK_ID, IDConverter
+ .toMRTaskAttemptId(taskAttemptId).toString());
+ jobConf.set(JobContext.TASK_ATTEMPT_ID,
+ IDConverter.toMRTaskAttemptId(taskAttemptId).toString());
jobConf.setInt(JobContext.TASK_PARTITION,
taskAttemptId.getTaskID().getId());
jobConf.set(JobContext.ID, taskAttemptId.getTaskID().getVertexID().getDAGId().toString());
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
index a6ab986..3558739 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java
@@ -69,11 +69,7 @@ public class MapProcessor extends MRTask implements Processor {
public void initialize(Configuration conf, Master master) throws IOException,
InterruptedException {
super.initialize(conf, master);
- TaskSplitMetaInfo[] allMetaInfo = readSplits();
- TaskSplitMetaInfo thisTaskMetaInfo = allMetaInfo[tezEngineTaskContext
- .getTaskAttemptId().getTaskID().getId()];
- splitMetaInfo = new TaskSplitIndex(thisTaskMetaInfo.getSplitLocation(),
- thisTaskMetaInfo.getStartOffset());
+
}
@Override
@@ -81,6 +77,14 @@ public class MapProcessor extends MRTask implements Processor {
final Input[] ins,
final Output[] outs)
throws IOException, InterruptedException {
+
+ // Read split information.
+ TaskSplitMetaInfo[] allMetaInfo = readSplits();
+ TaskSplitMetaInfo thisTaskMetaInfo = allMetaInfo[tezEngineTaskContext
+ .getTaskAttemptId().getTaskID().getId()];
+ splitMetaInfo = new TaskSplitIndex(thisTaskMetaInfo.getSplitLocation(),
+ thisTaskMetaInfo.getStartOffset());
+
MRTaskReporter reporter = new MRTaskReporter(getTaskReporter());
boolean useNewApi = jobConf.getUseNewMapper();
initTask(jobConf, taskAttemptId.getTaskID().getVertexID().getDAGId(),
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
index 025bf61..840cb31 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
@@ -111,7 +111,7 @@ public class TestUmbilicalProtocol implements TezTaskUmbilicalProtocol {
@Override
public boolean canCommit(TezTaskAttemptID taskid) throws IOException {
LOG.info("Got 'can-commit' from " + taskid);
- return false;
+ return true;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
index aec6667..0b6bc5f 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
@@ -18,40 +18,37 @@
package org.apache.tez.mapreduce.processor;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
-
import java.io.IOException;
-import java.util.Collections;
+import java.util.List;
import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex;
+import org.apache.hadoop.mapreduce.split.JobSplit;
+import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo;
+import org.apache.hadoop.mapreduce.split.SplitMetaInfoReaderTez;
import org.apache.tez.common.InputSpec;
import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.TezEngineTaskContext;
+import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
-import org.apache.tez.engine.api.Input;
-import org.apache.tez.engine.api.Output;
-import org.apache.tez.engine.api.Processor;
import org.apache.tez.engine.api.Task;
-import org.apache.tez.engine.lib.output.LocalOnFileSorterOutput;
import org.apache.tez.engine.runtime.RuntimeUtils;
-import org.apache.tez.engine.task.RuntimeTask;
import org.apache.tez.mapreduce.TezTestUtils;
-import org.apache.tez.mapreduce.input.SimpleInput;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
import org.apache.tez.mapreduce.processor.map.MapProcessor;
public class MapUtils {
@@ -62,8 +59,7 @@ public class MapUtils {
createInputSplit(FileSystem fs, Path workDir, JobConf job, Path file)
throws IOException {
FileInputFormat.setInputPaths(job, workDir);
-
-
+
// create a file with length entries
SequenceFile.Writer writer =
SequenceFile.createWriter(fs, job, file,
@@ -92,30 +88,59 @@ public class MapUtils {
"file = " + ((FileSplit)splits[0]).getPath());
return splits[0];
}
+
+ final private static FsPermission JOB_FILE_PERMISSION = FsPermission
+ .createImmutable((short) 0644); // rw-r--r--
+
+ // Will write files to PWD, from where they are read.
+
+ private static void writeSplitFiles(FileSystem fs, JobConf conf,
+ InputSplit split) throws IOException {
+ Path jobSplitFile = new Path(conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
+ TezJobConfig.DEFAULT_TASK_LOCAL_RESOURCE_DIR), MRJobConfig.JOB_SPLIT);
+ FSDataOutputStream out = FileSystem.create(fs, jobSplitFile,
+ new FsPermission(JOB_FILE_PERMISSION));
+
+ long offset = out.getPos();
+ Text.writeString(out, split.getClass().getName());
+ split.write(out);
+ out.close();
+
+ String[] locations = split.getLocations();
+
+ SplitMetaInfo info = null;
+ info = new JobSplit.SplitMetaInfo(locations, offset, split.getLength());
+
+ Path jobSplitMetaInfoFile = new Path(
+ conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR),
+ MRJobConfig.JOB_SPLIT_METAINFO);
+
+ FSDataOutputStream outMeta = FileSystem.create(fs, jobSplitMetaInfoFile,
+ new FsPermission(JOB_FILE_PERMISSION));
+ outMeta.write(SplitMetaInfoReaderTez.META_SPLIT_FILE_HEADER);
+ WritableUtils.writeVInt(outMeta, SplitMetaInfoReaderTez.META_SPLIT_VERSION);
+ WritableUtils.writeVInt(outMeta, 1); // Only 1 split meta info being written
+ info.write(outMeta);
+ outMeta.close();
+ }
public static Task runMapProcessor(FileSystem fs, Path workDir,
- JobConf jobConf,
- int mapId, Path mapInput,
+ JobConf jobConf, int mapId, Path mapInput,
TezTaskUmbilicalProtocol umbilical,
- Class<?> outputClazz) throws Exception {
+ String vertexName, List<InputSpec> inputSpecs,
+ List<OutputSpec> outputSpecs) throws Exception {
jobConf.setInputFormat(SequenceFileInputFormat.class);
InputSplit split = createInputSplit(fs, workDir, jobConf, mapInput);
- TezEngineTaskContext taskContext =
- new TezEngineTaskContext(
- TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0), "tez",
- "tez", "TODO_vertexName", MapProcessor.class.getName(),
- Collections.singletonList(new InputSpec("srcVertex", 0,
- SimpleInput.class.getName())),
- Collections.singletonList(new OutputSpec("targetVertex", 0,
- outputClazz.getName())));
+
+ writeSplitFiles(fs, jobConf, split);
+ TezEngineTaskContext taskContext = new TezEngineTaskContext(
+ TezTestUtils.getMockTaskAttemptId(0, 0, mapId, 0), "testuser",
+ "testJob", vertexName, MapProcessor.class.getName(),
+ inputSpecs, outputSpecs);
Task t = RuntimeUtils.createRuntimeTask(taskContext);
t.initialize(jobConf, umbilical);
- SimpleInput[] real = ((SimpleInput[])t.getInputs());
- SimpleInput[] inputs = spy(real);
- doReturn(split).when(inputs[0]).getOldSplitDetails(any(TaskSplitIndex.class));
- t.getProcessor().process(inputs, t.getOutputs());
+ t.getProcessor().process(t.getInputs(), t.getOutputs());
return t;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
index f09340c..cda15fb 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
@@ -19,9 +19,11 @@ package org.apache.tez.mapreduce.processor.map;
import java.io.IOException;
+import java.util.Collections;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
@@ -29,6 +31,8 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.tez.common.Constants;
+import org.apache.tez.common.InputSpec;
+import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.engine.api.Task;
import org.apache.tez.engine.common.sort.impl.IFile;
@@ -37,11 +41,15 @@ import org.apache.tez.engine.common.task.local.output.TezTaskOutput;
import org.apache.tez.engine.lib.output.InMemorySortedOutput;
import org.apache.tez.engine.lib.output.LocalOnFileSorterOutput;
import org.apache.tez.mapreduce.TestUmbilicalProtocol;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfToTezTranslator;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+import org.apache.tez.mapreduce.input.SimpleInput;
import org.apache.tez.mapreduce.processor.MapUtils;
import org.jboss.netty.buffer.BigEndianHeapChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.TruncatedChannelBuffer;
import org.jboss.netty.handler.stream.ChunkedStream;
+import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
@@ -49,9 +57,7 @@ import org.junit.Test;
public class TestMapProcessor {
- private static final Log LOG = LogFactory.getLog(TestMapProcessor.class);
-
- JobConf job;
+ private static final Log LOG = LogFactory.getLog(TestMapProcessor.class);
private static JobConf defaultConf = new JobConf();
private static FileSystem localFs = null;
@@ -63,32 +69,51 @@ public class TestMapProcessor {
throw new RuntimeException("init failure", e);
}
}
+ @SuppressWarnings("deprecation")
private static Path workDir =
new Path(new Path(System.getProperty("test.build.data", "/tmp")),
"TestMapProcessor").makeQualified(localFs);
TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
-
- @Before
- public void setUp() {
- job = new JobConf(defaultConf);
+
+ public void setUpJobConf(JobConf job) {
job.set(TezJobConfig.LOCAL_DIR, workDir.toString());
job.setClass(
Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
TezLocalTaskOutputFiles.class,
TezTaskOutput.class);
job.setNumReduceTasks(1);
- mapOutputs.setConf(job);
+ }
+
+ @Before
+ @After
+ public void cleanup() throws Exception {
+ localFs.delete(workDir, true);
}
@Test
- @Ignore
public void testMapProcessor() throws Exception {
- localFs.delete(workDir, true);
- MapUtils.runMapProcessor(
- localFs, workDir, job, 0, new Path(workDir, "map0"),
- new TestUmbilicalProtocol(),
- LocalOnFileSorterOutput.class).close();
+ String vertexName = MultiStageMRConfigUtil.getInitialMapVertexName();
+ JobConf jobConf = new JobConf(defaultConf);
+ setUpJobConf(jobConf);
+ TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
+ mapOutputs.setConf(jobConf);
+
+ Configuration conf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
+ Configuration stageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
+ vertexName);
+
+ JobConf job = new JobConf(stageConf);
+
+ job.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
+ "localized-resources").toUri().toString());
+
+ MapUtils.runMapProcessor(localFs, workDir, job, 0,
+ new Path(workDir, "map0"), new TestUmbilicalProtocol(), vertexName,
+ Collections.singletonList(new InputSpec("NullVertex", 0,
+ SimpleInput.class.getName())),
+ Collections.singletonList(new OutputSpec("FakeVertex", 1,
+ LocalOnFileSorterOutput.class.getName()))).close();
Path mapOutputFile = mapOutputs.getInputFile(0);
LOG.info("mapOutputFile = " + mapOutputFile);
@@ -115,16 +140,34 @@ public class TestMapProcessor {
@Test
@Ignore
public void testMapProcessorWithInMemSort() throws Exception {
+
+ String vertexName = MultiStageMRConfigUtil.getInitialMapVertexName();
+
final int partitions = 2;
- job.setNumReduceTasks(partitions);
- job.setInt(TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE, partitions);
+ JobConf jobConf = new JobConf(defaultConf);
+ jobConf.setNumReduceTasks(partitions);
+ setUpJobConf(jobConf);
+ TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
+ mapOutputs.setConf(jobConf);
+
+ Configuration conf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
+ Configuration stageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
+ vertexName);
+
+ JobConf job = new JobConf(stageConf);
+ job.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
+ "localized-resources").toUri().toString());
localFs.delete(workDir, true);
Task t =
MapUtils.runMapProcessor(
localFs, workDir, job, 0, new Path(workDir, "map0"),
- new TestUmbilicalProtocol(true),
- InMemorySortedOutput.class);
+ new TestUmbilicalProtocol(true), vertexName,
+ Collections.singletonList(new InputSpec("NullVertex", 0,
+ SimpleInput.class.getName())),
+ Collections.singletonList(new OutputSpec("FakeVertex", 1,
+ InMemorySortedOutput.class.getName()))
+ );
InMemorySortedOutput[] outputs = (InMemorySortedOutput[])t.getOutputs();
verifyInMemSortedStream(outputs[0], 0, 4096);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cee79f63/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
index afc21a7..69571e1 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
@@ -22,40 +22,44 @@ import java.util.Collections;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.tez.common.Constants;
import org.apache.tez.common.InputSpec;
import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.TezEngineTaskContext;
import org.apache.tez.common.TezJobConfig;
-import org.apache.tez.engine.api.Input;
-import org.apache.tez.engine.api.Output;
import org.apache.tez.engine.api.Task;
import org.apache.tez.engine.common.task.local.output.TezLocalTaskOutputFiles;
import org.apache.tez.engine.common.task.local.output.TezTaskOutput;
import org.apache.tez.engine.lib.input.LocalMergedInput;
import org.apache.tez.engine.lib.output.LocalOnFileSorterOutput;
import org.apache.tez.engine.runtime.RuntimeUtils;
-import org.apache.tez.engine.task.RuntimeTask;
import org.apache.tez.mapreduce.TestUmbilicalProtocol;
import org.apache.tez.mapreduce.TezTestUtils;
+import org.apache.tez.mapreduce.hadoop.IDConverter;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfToTezTranslator;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+import org.apache.tez.mapreduce.input.SimpleInput;
import org.apache.tez.mapreduce.output.SimpleOutput;
import org.apache.tez.mapreduce.processor.MapUtils;
+import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
public class TestReduceProcessor {
private static final Log LOG = LogFactory.getLog(TestReduceProcessor.class);
-
- JobConf job;
-
+
private static JobConf defaultConf = new JobConf();
private static FileSystem localFs = null;
static {
@@ -66,13 +70,12 @@ public class TestReduceProcessor {
throw new RuntimeException("init failure", e);
}
}
+ @SuppressWarnings("deprecation")
private static Path workDir =
new Path(new Path(System.getProperty("test.build.data", "/tmp")),
"TestReduceProcessor").makeQualified(localFs);
- @Before
- public void setUp() {
- job = new JobConf(defaultConf);
+ public void setUpJobConf(JobConf job) {
job.set(TezJobConfig.LOCAL_DIR, workDir.toString());
job.setClass(
Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
@@ -80,37 +83,84 @@ public class TestReduceProcessor {
TezTaskOutput.class);
job.setNumReduceTasks(1);
}
-
+
+ @Before
+ @After
+ public void cleanup() throws Exception {
+ localFs.delete(workDir, true);
+ }
+
@Test
- @Ignore
public void testReduceProcessor() throws Exception {
- localFs.delete(workDir, true);
+ String mapVertexName = MultiStageMRConfigUtil.getInitialMapVertexName();
+ String reduceVertexName = MultiStageMRConfigUtil.getFinalReduceVertexName();
+ JobConf jobConf = new JobConf(defaultConf);
+ setUpJobConf(jobConf);
+ TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
+ mapOutputs.setConf(jobConf);
+
+ Configuration conf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
+ Configuration mapStageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
+ mapVertexName);
+
+ JobConf mapConf = new JobConf(mapStageConf);
+
+ mapConf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
+ "localized-resources").toUri().toString());
+
// Run a map
- MapUtils.runMapProcessor(
- localFs, workDir, job, 0, new Path(workDir, "map0"),
- new TestUmbilicalProtocol(),
- LocalOnFileSorterOutput.class);
+ MapUtils.runMapProcessor(localFs, workDir, mapConf, 0,
+ new Path(workDir, "map0"), new TestUmbilicalProtocol(), mapVertexName,
+ Collections.singletonList(new InputSpec("NullVertex", 0,
+ SimpleInput.class.getName())),
+ Collections.singletonList(new OutputSpec("FakeVertex", 1,
+ LocalOnFileSorterOutput.class.getName())));
LOG.info("Starting reduce...");
- FileOutputFormat.setOutputPath(job, new Path(workDir, "output"));
+
+
+ Configuration reduceStageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
+ reduceVertexName);
+ JobConf reduceConf = new JobConf(reduceStageConf);
+ reduceConf.setOutputFormat(SequenceFileOutputFormat.class);
+ FileOutputFormat.setOutputPath(reduceConf, new Path(workDir, "output"));
// Now run a reduce
TezEngineTaskContext taskContext = new TezEngineTaskContext(
- TezTestUtils.getMockTaskAttemptId(0, 0, 0, 0), "tez",
- "tez", "TODO_vertexName", ReduceProcessor.class.getName(),
- Collections.singletonList(new InputSpec("TODO_srcVertexName", 1,
+ TezTestUtils.getMockTaskAttemptId(0, 1, 0, 0), "testUser",
+ "testJob", reduceVertexName, ReduceProcessor.class.getName(),
+ Collections.singletonList(new InputSpec(mapVertexName, 1,
LocalMergedInput.class.getName())),
- Collections.singletonList(new OutputSpec("TODO_targetVertexName", 1,
+ Collections.singletonList(new OutputSpec("", 1,
SimpleOutput.class.getName())));
-
- job.set(JobContext.TASK_ATTEMPT_ID,
- taskContext.getTaskAttemptId().toString());
+
Task t = RuntimeUtils.createRuntimeTask(taskContext);
- t.initialize(job, new TestUmbilicalProtocol());
+ t.initialize(reduceConf, new TestUmbilicalProtocol());
t.run();
t.close();
+
+ // Can this be done via some utility class ? MapOutputFile derivative, or
+ // instantiating the OutputCommitter
+ Path reduceOutputDir = new Path(new Path(workDir, "output"),
+ "_temporary/0/" + IDConverter
+ .toMRTaskId(taskContext.getTaskAttemptId().getTaskID()));
+ Path reduceOutputFile = new Path(reduceOutputDir, "part-00000");
+
+ @SuppressWarnings("deprecation")
+ SequenceFile.Reader reader = new SequenceFile.Reader(localFs, reduceOutputFile, reduceConf);
+
+ LongWritable key = new LongWritable();
+ Text value = new Text();
+ long prev = Long.MIN_VALUE;
+ while (reader.next(key, value)) {
+ if (prev != Long.MIN_VALUE) {
+ Assert.assertTrue(prev < key.get());
+ prev = key.get();
+ }
+ }
+ reader.close();
}
}
[24/50] [abbrv] git commit: TEZ-153. Fix MAPREDUCE-5156 build break
(bikas)
Posted by ss...@apache.org.
TEZ-153. Fix MAPREDUCE-5156 build break (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/36198a63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/36198a63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/36198a63
Branch: refs/heads/master
Commit: 36198a636d2db65fb91934725dc6759d5dda231f
Parents: 7d401d2
Author: Bikas Saha <bi...@apache.org>
Authored: Wed May 22 16:14:43 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Wed May 22 16:14:43 2013 -0700
----------------------------------------------------------------------
.../tez/mapreduce/examples/ExampleDriver.java | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/36198a63/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
index 0e4055a..94ba79f 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
@@ -61,7 +61,7 @@ public class ExampleDriver {
+ " containing employee_name department name per line of input"
+ " and generates count of employees per department and"
+ " sorted on employee count");
- exitCode = pgd.driver(argv);
+ exitCode = pgd.run(argv);
}
catch(Throwable e){
e.printStackTrace();
[18/50] [abbrv] git commit: TEZ-111. Create tests for DAGImpl.
(hitesh)
Posted by ss...@apache.org.
TEZ-111. Create tests for DAGImpl. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/8298190d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/8298190d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/8298190d
Branch: refs/heads/master
Commit: 8298190ddb513652ea65f72a5104f6b3b3320c86
Parents: be6d4bc
Author: Hitesh Shah <hi...@apache.org>
Authored: Mon May 20 14:49:43 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Mon May 20 14:49:43 2013 -0700
----------------------------------------------------------------------
.../apache/tez/dag/app/dag/event/DAGEventType.java | 17 +-
.../org/apache/tez/dag/app/dag/impl/DAGImpl.java | 45 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 76 +-
.../apache/tez/dag/app/dag/impl/TestDAGImpl.java | 626 +++++++++++++++
.../tez/dag/app/dag/impl/TestVertexImpl.java | 167 +++-
5 files changed, 814 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8298190d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
index 20f14b2..14c2f30 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java
@@ -30,24 +30,17 @@ public enum DAGEventType {
DAG_INIT,
DAG_START,
- //Producer:Task
- /*
- JOB_TASK_COMPLETED,
- JOB_MAP_TASK_RESCHEDULED,
- JOB_TASK_ATTEMPT_COMPLETED,
- */
-
//Producer: Vertex
- DAG_VERTEX_INITED,
- DAG_VERTEX_STARTED,
DAG_VERTEX_COMPLETED,
+
+ //Producer: TaskImpl
DAG_SCHEDULER_UPDATE,
-
- //Producer:Job
+
+ //Producer:Dag
DAG_COMPLETED,
//Producer:Any component
DAG_DIAGNOSTIC_UPDATE,
INTERNAL_ERROR,
- DAG_COUNTER_UPDATE,
+ DAG_COUNTER_UPDATE,
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8298190d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 66f9c36..5f47818 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -117,7 +117,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private final TaskAttemptListener taskAttemptListener;
private final TaskHeartbeatHandler taskHeartbeatHandler;
private final Object tasksSyncHandle = new Object();
-
+
private DAGScheduler dagScheduler;
private final EventHandler eventHandler;
@@ -129,7 +129,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private final AppContext appContext;
volatile Map<TezVertexID, Vertex> vertices = new HashMap<TezVertexID, Vertex>();
- private Map<String, EdgeProperty> edges = new HashMap<String, EdgeProperty>();
+ private Map<String, EdgeProperty> edges = new HashMap<String, EdgeProperty>();
private TezCounters dagCounters = new TezCounters();
private Object fullCountersLock = new Object();
private TezCounters fullCounters = null;
@@ -150,7 +150,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
INTERNAL_ERROR_TRANSITION = new InternalErrorTransition();
private static final CounterUpdateTransition COUNTER_UPDATE_TRANSITION =
new CounterUpdateTransition();
- private static final DAGSchedulerUpdateTransition
+ private static final DAGSchedulerUpdateTransition
DAG_SCHEDULER_UPDATE_TRANSITION = new DAGSchedulerUpdateTransition();
protected static final
@@ -211,8 +211,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
DIAGNOSTIC_UPDATE_TRANSITION)
.addTransition(DAGState.RUNNING, DAGState.RUNNING,
DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
- .addTransition(DAGState.RUNNING, DAGState.RUNNING,
- DAGEventType.DAG_SCHEDULER_UPDATE,
+ .addTransition(DAGState.RUNNING, DAGState.RUNNING,
+ DAGEventType.DAG_SCHEDULER_UPDATE,
DAG_SCHEDULER_UPDATE_TRANSITION)
.addTransition(
DAGState.RUNNING,
@@ -332,7 +332,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
this.jobPlan = jobPlan;
this.conf = conf;
this.dagName = (jobPlan.getName() != null) ? jobPlan.getName() : "<missing app name>";
-
+
this.userName = appUserName;
// TODO Metrics
//this.metrics = metrics;
@@ -374,7 +374,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
public TezConfiguration getConf() {
return conf;
}
-
+
@Override
public DAGPlan getJobPlan() {
return jobPlan;
@@ -508,7 +508,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
readLock.unlock();
}
}
-
+
// monitoring apis
@Override
public DAGStatusBuilder getDAGStatus() {
@@ -552,11 +552,12 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
return vertex.getVertexStatus();
}
-
-
+
+
protected void startRootVertices() {
for (Vertex v : vertices.values()) {
if (v.getInputVerticesCount() == 0) {
+ LOG.info("DEBUG: Starting root vertex " + v.getName());
eventHandler.handle(new VertexEvent(v.getVertexId(),
VertexEventType.V_START));
}
@@ -605,7 +606,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
writeLock.unlock();
}
}
-
+
@Private
public DAGState getInternalState() {
readLock.lock();
@@ -727,7 +728,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
public String getUserName() {
return userName;
}
-
+
@Override
public String getQueueName() {
return queueName;
@@ -862,7 +863,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
dag.edges = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dag.getJobPlan().getEdgeList());
Map<String,EdgePlan> edgePlans = DagTypeConverters.createEdgePlanMapFromDAGPlan(dag.getJobPlan().getEdgeList());
-
+
// setup the dag
for (Vertex v : dag.vertices.values()) {
parseVertexEdges(dag, edgePlans, v);
@@ -888,10 +889,10 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
TezVertexID vertexId = TezBuilderUtils.newVertexID(dag.getID(), vId);
-
+
VertexPlan vertexPlan = dag.getJobPlan().getVertex(vId);
VertexLocationHint vertexLocationHint = DagTypeConverters.convertFromDAGPlan(vertexPlan.getTaskLocationHintList());
-
+
return new VertexImpl(
vertexId, vertexPlan, vertexName, dag.conf,
dag.eventHandler, dag.taskAttemptListener,
@@ -912,18 +913,18 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
for(String inEdgeId : vertexPlan.getInEdgeIdList()){
EdgePlan edgePlan = edgePlans.get(inEdgeId);
- Vertex inVertex = dag.vertexMap.get(edgePlan.getInputVertexName());
+ Vertex inVertex = dag.vertexMap.get(edgePlan.getInputVertexName());
EdgeProperty edgeProp = dag.edges.get(inEdgeId);
inVertices.put(inVertex, edgeProp);
}
-
+
for(String outEdgeId : vertexPlan.getOutEdgeIdList()){
EdgePlan edgePlan = edgePlans.get(outEdgeId);
- Vertex outVertex = dag.vertexMap.get(edgePlan.getOutputVertexName());
+ Vertex outVertex = dag.vertexMap.get(edgePlan.getOutputVertexName());
EdgeProperty edgeProp = dag.edges.get(outEdgeId);
outVertices.put(outVertex, edgeProp);
}
-
+
vertex.setInputVertices(inVertices);
vertex.setOutputVertices(outVertices);
}
@@ -931,9 +932,9 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
protected void setup(DAGImpl job) throws IOException {
job.initTime = job.clock.getTime();
String dagIdString = job.dagId.toString();
-
+
dagIdString.replace("application", "job");
-
+
// TODO remove - TEZ-71
String user =
UserGroupInformation.getCurrentUser().getShortUserName();
@@ -1165,7 +1166,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
}
}
-
+
private static class DAGSchedulerUpdateTransition implements
SingleArcTransition<DAGImpl, DAGEvent> {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8298190d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 20d13cd..ce4c609 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -141,7 +141,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// TODO Metrics
//private final MRAppMetrics metrics;
private final AppContext appContext;
-
+
private boolean lazyTasksCopyNeeded = false;
volatile Map<TezTaskID, Task> tasks = new LinkedHashMap<TezTaskID, Task>();
private Object fullCountersLock = new Object();
@@ -154,7 +154,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private int numStartedSourceVertices = 0;
private int distanceFromRoot = 0;
-
+
private List<TezDependentTaskCompletionEvent> sourceTaskAttemptCompletionEvents;
private final List<String> diagnostics = new ArrayList<String>();
@@ -167,7 +167,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
List<InputSpec> inputSpecList;
List<OutputSpec> outputSpecList;
-
+
private static final InternalErrorTransition
INTERNAL_ERROR_TRANSITION = new InternalErrorTransition();
private static final TaskAttemptCompletedEventTransition
@@ -200,7 +200,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
.addTransition(VertexState.INITED, VertexState.INITED,
VertexEventType.V_SOURCE_VERTEX_STARTED,
new SourceVertexStartedTransition())
- .addTransition(VertexState.INITED, VertexState.RUNNING,
+ .addTransition(VertexState.INITED, VertexState.RUNNING,
VertexEventType.V_START,
new StartTransition())
@@ -220,7 +220,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
.addTransition
(VertexState.RUNNING,
- EnumSet.of(VertexState.RUNNING, VertexState.KILLED,
+ EnumSet.of(VertexState.RUNNING,
VertexState.SUCCEEDED, VertexState.FAILED),
VertexEventType.V_TASK_COMPLETED,
new TaskCompletedTransition())
@@ -335,14 +335,14 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private Token<JobTokenIdentifier> jobToken;
private final TezVertexID vertexId; //runtime assigned id.
- private final VertexPlan vertexPlan;
+ private final VertexPlan vertexPlan;
private final String vertexName;
private final String processorName;
private Map<Vertex, EdgeProperty> sourceVertices;
private Map<Vertex, EdgeProperty> targetVertices;
-
+
private VertexScheduler vertexScheduler;
private VertexOutputCommitter committer;
@@ -351,8 +351,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private Map<String, LocalResource> localResources;
private Map<String, String> environment;
private final String javaOpts;
-
- public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan,
+
+ public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan,
String vertexName, TezConfiguration conf, EventHandler eventHandler,
TaskAttemptListener taskAttemptListener,
Token<JobTokenIdentifier> jobToken,
@@ -384,13 +384,13 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
this.jobToken = jobToken;
this.committer = new NullVertexOutputCommitter();
this.vertexLocationHint = vertexLocationHint;
-
+
this.taskResource = DagTypeConverters.CreateResourceRequestFromTaskConfig(vertexPlan.getTaskConfig());
- this.processorName = vertexPlan.hasProcessorName() ? vertexPlan.getProcessorName() : null;
+ this.processorName = vertexPlan.hasProcessorName() ? vertexPlan.getProcessorName() : null;
this.localResources = DagTypeConverters.createLocalResourceMapFromDAGPlan(vertexPlan.getTaskConfig().getLocalResourceList());
this.environment = DagTypeConverters.createEnvironmentMapFromDAGPlan(vertexPlan.getTaskConfig().getEnvironmentSettingList());
this.javaOpts = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan.getTaskConfig().getJavaOpts() : null;
-
+
// This "this leak" is okay because the retained pointer is in an
// instance variable.
stateMachine = stateMachineFactory.make(this);
@@ -404,12 +404,12 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
public TezVertexID getVertexId() {
return vertexId;
}
-
+
@Override
public VertexPlan getVertexPlan() {
return vertexPlan;
}
-
+
@Override
public int getDistanceFromRoot() {
return distanceFromRoot;
@@ -521,7 +521,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
this.readLock.unlock();
}
}
-
+
@Override
public ProgressBuilder getVertexProgress() {
this.readLock.lock();
@@ -537,7 +537,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
this.readLock.unlock();
}
}
-
+
@Override
public VertexStatusBuilder getVertexStatus() {
this.readLock.lock();
@@ -631,7 +631,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
writeLock.unlock();
}
}
-
+
private VertexState getInternalState() {
readLock.lock();
try {
@@ -710,7 +710,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
vertex.abortVertex(VertexStatus.State.FAILED);
return vertex.finished(VertexState.FAILED);
}
-
+
if(vertex.succeededTaskCount == vertex.tasks.size()) {
try {
if (!vertex.committed.getAndSet(true)) {
@@ -721,9 +721,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
LOG.error("Failed to do commit on vertex, name=" + vertex.getName(), e);
return vertex.finished(VertexState.FAILED);
}
- return vertex.finished(VertexState.SUCCEEDED);
+ return vertex.finished(VertexState.SUCCEEDED);
}
-
+
if (vertex.completedTaskCount == vertex.tasks.size()) {
// this means the vertex has some killed tasks
assert vertex.killedTaskCount > 0;
@@ -731,9 +731,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
vertex.abortVertex(VertexStatus.State.KILLED);
return vertex.finished(VertexState.KILLED);
}
-
+
//return the current state, Vertex not finished yet
- return vertex.getInternalState();
+ return vertex.getInternalState();
}
VertexState finished(VertexState finalState) {
@@ -774,9 +774,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// TODODAGAM
// TODO: Splits?
-
+
vertex.numTasks = vertex.getVertexPlan().getTaskConfig().getNumTasks();
-
+
/*
TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(job, job.jobId);
job.numMapTasks = taskSplitMetaInfo.length;
@@ -794,8 +794,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// create the Tasks but don't start them yet
createTasks(vertex);
-
-
+
+
boolean hasBipartite = false;
if (vertex.sourceVertices != null) {
@@ -806,10 +806,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
}
}
-
+
if (hasBipartite) {
// setup vertex scheduler
- // TODO this needs to consider data size and perhaps API.
+ // TODO this needs to consider data size and perhaps API.
// Currently implicitly BIPARTITE is the only edge type
vertex.vertexScheduler = new BipartiteSlowStartVertexScheduler(
vertex,
@@ -907,7 +907,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
@Override
public void transition(VertexImpl vertex, VertexEvent event) {
- VertexEventSourceVertexStarted startEvent =
+ VertexEventSourceVertexStarted startEvent =
(VertexEventSourceVertexStarted) event;
int distanceFromRoot = startEvent.getSourceDistanceFromRoot() + 1;
if(vertex.distanceFromRoot < distanceFromRoot) {
@@ -916,8 +916,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
vertex.numStartedSourceVertices++;
if (vertex.numStartedSourceVertices == vertex.sourceVertices.size()) {
// Consider inlining this.
- LOG.info("Starting vertex: " + vertex.getVertexId() +
- " with name: " + vertex.getName() +
+ LOG.info("Starting vertex: " + vertex.getVertexId() +
+ " with name: " + vertex.getName() +
" with distanceFromRoot: " + vertex.distanceFromRoot );
vertex.eventHandler.handle(new VertexEvent(vertex.vertexId,
VertexEventType.V_START));
@@ -1040,10 +1040,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
//eventId is equal to index in the arraylist
tce.setEventId(vertex.sourceTaskAttemptCompletionEvents.size());
vertex.sourceTaskAttemptCompletionEvents.add(tce);
- // TODO this needs to be ordered/grouped by source vertices or else
- // my tasks will not know which events are for which vertices' tasks. This
+ // TODO this needs to be ordered/grouped by source vertices or else
+ // my tasks will not know which events are for which vertices' tasks. This
// differentiation was not needed for MR because there was only 1 M stage.
- // if the tce is sent to the task then a solution could be to add vertex
+ // if the tce is sent to the task then a solution could be to add vertex
// name to the tce
// need to send vertex name and task index in that vertex
@@ -1060,7 +1060,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
vertex.successSourceAttemptCompletionEventNoMap.put(taskId, tce.getEventId());
}
-
+
vertex.vertexScheduler.onSourceTaskCompleted(attemptId);
}
}
@@ -1073,7 +1073,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
TezDependentTaskCompletionEvent tce =
((VertexEventTaskAttemptCompleted) event).getCompletionEvent();
- // TODO this should only be sent for successful events? looks like all
+ // TODO this should only be sent for successful events? looks like all
// need to be sent in the existing shuffle code
// Notify all target vertices
if (vertex.targetVertices != null) {
@@ -1245,7 +1245,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
public TezDAGID getDAGId() {
return getDAG().getID();
}
-
+
@Override
public ApplicationAttemptId getApplicationAttemptId() {
return appContext.getApplicationAttemptId();
@@ -1259,7 +1259,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
public DAG getDAG() {
return appContext.getDAG();
}
-
+
@VisibleForTesting
String getProcessorName() {
return this.processorName;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8298190d/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
new file mode 100644
index 0000000..3adcd8b
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -0,0 +1,626 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.SystemClock;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.oldrecords.TaskState;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeConnectionPattern;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeSourceType;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskHeartbeatHandler;
+import org.apache.tez.dag.app.dag.DAGState;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.VertexState;
+import org.apache.tez.dag.app.dag.event.DAGEvent;
+import org.apache.tez.dag.app.dag.event.DAGEventType;
+import org.apache.tez.dag.app.dag.event.DAGEventVertexCompleted;
+import org.apache.tez.dag.app.dag.event.DAGFinishEvent;
+import org.apache.tez.dag.app.dag.event.TaskEvent;
+import org.apache.tez.dag.app.dag.event.TaskEventType;
+import org.apache.tez.dag.app.dag.event.VertexEvent;
+import org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted;
+import org.apache.tez.dag.app.dag.event.VertexEventType;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.avro.HistoryEventType;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.engine.common.security.JobTokenSecretManager;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestDAGImpl {
+
+ private static final Log LOG = LogFactory.getLog(TestDAGImpl.class);
+ private DAGPlan dagPlan;
+ private TezDAGID dagId;
+ private TezConfiguration conf;
+ private DrainDispatcher dispatcher;
+ private Credentials fsTokens;
+ private AppContext appContext;
+ private ApplicationAttemptId appAttemptId;
+ private DAGImpl dag;
+ private VertexEventDispatcher vertexEventDispatcher;
+ private DagEventDispatcher dagEventDispatcher;
+ private TaskAttemptListener taskAttemptListener;
+ private TaskHeartbeatHandler thh;
+ private Clock clock = new SystemClock();
+ private JobTokenSecretManager jobTokenSecretManager;
+ private DAGFinishEventHandler dagFinishEventHandler;
+
+ private class DagEventDispatcher implements EventHandler<DAGEvent> {
+ @Override
+ public void handle(DAGEvent event) {
+ dag.handle(event);
+ }
+ }
+
+ private class HistoryHandler implements EventHandler<DAGHistoryEvent> {
+ @Override
+ public void handle(DAGHistoryEvent event) {
+ }
+ }
+
+ private class TaskEventHandler implements EventHandler<TaskEvent> {
+ @Override
+ public void handle(TaskEvent event) {
+ }
+ }
+
+ private class VertexEventDispatcher
+ implements EventHandler<VertexEvent> {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void handle(VertexEvent event) {
+ Vertex vertex = dag.getVertex(event.getVertexId());
+ ((EventHandler<VertexEvent>) vertex).handle(event);
+ }
+ }
+
+ private class DAGFinishEventHandler
+ implements EventHandler<DAGFinishEvent> {
+ public int dagFinishEvents = 0;
+
+ @Override
+ public void handle(DAGFinishEvent event) {
+ ++dagFinishEvents;
+ }
+ }
+
+ private DAGPlan createTestDAGPlan() {
+ LOG.info("Setting up dag plan");
+ DAGPlan dag = DAGPlan.newBuilder()
+ .setName("testverteximpl")
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex1")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host1")
+ .addRack("rack1")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(1)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x1.y1")
+ .build()
+ )
+ .addOutEdgeId("e1")
+ .build()
+ )
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex2")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host2")
+ .addRack("rack2")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x2.y2")
+ .build()
+ )
+ .addOutEdgeId("e2")
+ .build()
+ )
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex3")
+ .setType(PlanVertexType.NORMAL)
+ .setProcessorName("x3.y3")
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host3")
+ .addRack("rack3")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("foo")
+ .setTaskModule("x3.y3")
+ .build()
+ )
+ .addInEdgeId("e1")
+ .addInEdgeId("e2")
+ .addOutEdgeId("e3")
+ .addOutEdgeId("e4")
+ .build()
+ )
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex4")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host4")
+ .addRack("rack4")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x4.y4")
+ .build()
+ )
+ .addInEdgeId("e3")
+ .addOutEdgeId("e5")
+ .build()
+ )
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex5")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host5")
+ .addRack("rack5")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x5.y5")
+ .build()
+ )
+ .addInEdgeId("e4")
+ .addOutEdgeId("e6")
+ .build()
+ )
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex6")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host6")
+ .addRack("rack6")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x6.y6")
+ .build()
+ )
+ .addInEdgeId("e5")
+ .addInEdgeId("e6")
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i3_v1")
+ .setInputVertexName("vertex1")
+ .setOutputClass("o1")
+ .setOutputVertexName("vertex3")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e1")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i3_v2")
+ .setInputVertexName("vertex2")
+ .setOutputClass("o2")
+ .setOutputVertexName("vertex3")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e2")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i4_v3")
+ .setInputVertexName("vertex3")
+ .setOutputClass("o3_v4")
+ .setOutputVertexName("vertex4")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e3")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i5_v3")
+ .setInputVertexName("vertex3")
+ .setOutputClass("o3_v5")
+ .setOutputVertexName("vertex5")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e4")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i6_v4")
+ .setInputVertexName("vertex4")
+ .setOutputClass("o4")
+ .setOutputVertexName("vertex6")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e5")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i6_v5")
+ .setInputVertexName("vertex5")
+ .setOutputClass("o5")
+ .setOutputVertexName("vertex6")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e6")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .build();
+
+ return dag;
+ }
+
+ @Before
+ public void setup() {
+ conf = new TezConfiguration();
+ appAttemptId = BuilderUtils.newApplicationAttemptId(
+ BuilderUtils.newApplicationId(100, 1), 1);
+ dagId = new TezDAGID(appAttemptId.getApplicationId(), 1);
+ Assert.assertNotNull(dagId);
+ dagPlan = createTestDAGPlan();
+ dispatcher = new DrainDispatcher();
+ fsTokens = new Credentials();
+ jobTokenSecretManager = new JobTokenSecretManager();
+ appContext = mock(AppContext.class);
+ doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
+ doReturn(dagId).when(appContext).getDAGID();
+ dag = new DAGImpl(dagId, appAttemptId, conf, dagPlan,
+ dispatcher.getEventHandler(), taskAttemptListener,
+ jobTokenSecretManager, fsTokens, clock, "user", 10000, thh, appContext);
+ doReturn(dag).when(appContext).getDAG();
+ vertexEventDispatcher = new VertexEventDispatcher();
+ dispatcher.register(VertexEventType.class, vertexEventDispatcher);
+ dagEventDispatcher = new DagEventDispatcher();
+ dispatcher.register(DAGEventType.class, dagEventDispatcher);
+ dispatcher.register(HistoryEventType.class,
+ new HistoryHandler());
+ dagFinishEventHandler = new DAGFinishEventHandler();
+ dispatcher.register(DAGFinishEvent.Type.class, dagFinishEventHandler);
+ dispatcher.register(TaskEventType.class, new TaskEventHandler());
+ dispatcher.init(conf);
+ dispatcher.start();
+ }
+
+ @After
+ public void teardown() {
+ dagPlan = null;
+ dag = null;
+ dispatcher.await();
+ dispatcher.stop();
+ }
+
+ private void initDAG(DAGImpl dag) {
+ dag.handle(new DAGEvent(dagId, DAGEventType.DAG_INIT));
+ Assert.assertEquals(DAGState.INITED, dag.getState());
+ }
+
+ private void startDAG(DAGImpl dag) {
+ dag.handle(new DAGEvent(dagId, DAGEventType.DAG_START));
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+ }
+
+ @Test
+ public void testDAGInit() {
+ initDAG(dag);
+ Assert.assertEquals(6, dag.getTotalVertices());
+ }
+
+ @Test
+ public void testDAGStart() {
+ initDAG(dag);
+ startDAG(dag);
+ dispatcher.await();
+
+ for (int i = 0 ; i < 6; ++i ) {
+ TezVertexID vId = new TezVertexID(dagId, i);
+ Vertex v = dag.getVertex(vId);
+ Assert.assertEquals(VertexState.RUNNING, v.getState());
+ if (i < 2) {
+ Assert.assertEquals(0, v.getDistanceFromRoot());
+ } else if (i == 2) {
+ Assert.assertEquals(1, v.getDistanceFromRoot());
+ } else if ( i > 2 && i < 5) {
+ Assert.assertEquals(2, v.getDistanceFromRoot());
+ } else if (i == 5) {
+ Assert.assertEquals(3, v.getDistanceFromRoot());
+ }
+ }
+
+ for (int i = 0 ; i < 6; ++i ) {
+ TezVertexID vId = new TezVertexID(dagId, i);
+ LOG.info("Distance from root: v" + i + ":"
+ + dag.getVertex(vId).getDistanceFromRoot());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testVertexCompletion() {
+ initDAG(dag);
+ startDAG(dag);
+ dispatcher.await();
+
+ TezVertexID vId = new TezVertexID(dagId, 1);
+ Vertex v = dag.getVertex(vId);
+ ((EventHandler<VertexEvent>) v).handle(new VertexEventTaskCompleted(
+ new TezTaskID(vId, 0), TaskState.SUCCEEDED));
+ ((EventHandler<VertexEvent>) v).handle(new VertexEventTaskCompleted(
+ new TezTaskID(vId, 1), TaskState.SUCCEEDED));
+ dispatcher.await();
+
+ Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+ Assert.assertEquals(1, dag.getSuccessfulVertices());
+ }
+
+ public void testKillStartedDAG() {
+ initDAG(dag);
+ startDAG(dag);
+ dispatcher.await();
+
+ dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+ dispatcher.await();
+
+ Assert.assertEquals(DAGState.KILLED, dag.getState());
+ for (int i = 0 ; i < 6; ++i ) {
+ TezVertexID vId = new TezVertexID(dagId, i);
+ Vertex v = dag.getVertex(vId);
+ Assert.assertEquals(VertexState.KILLED, v.getState());
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testKillRunningDAG() {
+ initDAG(dag);
+ startDAG(dag);
+ dispatcher.await();
+
+ TezVertexID vId1 = new TezVertexID(dagId, 1);
+ Vertex v1 = dag.getVertex(vId1);
+ ((EventHandler<VertexEvent>) v1).handle(new VertexEventTaskCompleted(
+ new TezTaskID(vId1, 0), TaskState.SUCCEEDED));
+ TezVertexID vId0 = new TezVertexID(dagId, 0);
+ Vertex v0 = dag.getVertex(vId0);
+ ((EventHandler<VertexEvent>) v0).handle(new VertexEventTaskCompleted(
+ new TezTaskID(vId0, 0), TaskState.SUCCEEDED));
+ dispatcher.await();
+
+ Assert.assertEquals(VertexState.SUCCEEDED, v0.getState());
+ Assert.assertEquals(VertexState.RUNNING, v1.getState());
+
+ dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+ dispatcher.await();
+
+ Assert.assertEquals(DAGState.KILL_WAIT, dag.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v0.getState());
+ Assert.assertEquals(VertexState.KILL_WAIT, v1.getState());
+ for (int i = 2 ; i < 6; ++i ) {
+ TezVertexID vId = new TezVertexID(dagId, i);
+ Vertex v = dag.getVertex(vId);
+ Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+ }
+ Assert.assertEquals(1, dag.getSuccessfulVertices());
+ }
+
+ @Test
+ public void testInvalidEvent() {
+ dag.handle(new DAGEvent(dagId, DAGEventType.DAG_START));
+ dispatcher.await();
+ Assert.assertEquals(DAGState.ERROR, dag.getState());
+ }
+
+ @Test
+ @Ignore
+ public void testVertexSuccessfulCompletionUpdates() {
+ initDAG(dag);
+ startDAG(dag);
+ dispatcher.await();
+
+ for (int i = 0; i < 6; ++i) {
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
+ }
+ dispatcher.await();
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+ Assert.assertEquals(1, dag.getSuccessfulVertices());
+
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 2), VertexState.SUCCEEDED));
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 3), VertexState.SUCCEEDED));
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 4), VertexState.SUCCEEDED));
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 5), VertexState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(DAGState.SUCCEEDED, dag.getState());
+ Assert.assertEquals(6, dag.getSuccessfulVertices());
+ }
+
+ @Test
+ @Ignore
+ public void testVertexFailureHandling() {
+ initDAG(dag);
+ startDAG(dag);
+ dispatcher.await();
+
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 2), VertexState.FAILED));
+ dispatcher.await();
+ Assert.assertEquals(DAGState.FAILED, dag.getState());
+ Assert.assertEquals(2, dag.getSuccessfulVertices());
+
+ // Expect running vertices to be killed on first failure
+ for (int i = 3; i < 6; ++i) {
+ TezVertexID vId = new TezVertexID(dagId, i);
+ Vertex v = dag.getVertex(vId);
+ Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+ }
+ }
+
+ @Test
+ @Ignore
+ public void testDAGKill() {
+ initDAG(dag);
+ startDAG(dag);
+ dispatcher.await();
+
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
+ dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+
+ for (int i = 2; i < 6; ++i) {
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, i), VertexState.SUCCEEDED));
+ }
+ dispatcher.await();
+ Assert.assertEquals(DAGState.KILLED, dag.getState());
+ Assert.assertEquals(6, dag.getSuccessfulVertices());
+ Assert.assertEquals(1, dagFinishEventHandler.dagFinishEvents);
+ }
+
+ @Test
+ public void testDAGKillPending() {
+ initDAG(dag);
+ startDAG(dag);
+ dispatcher.await();
+
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 0), VertexState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(DAGState.RUNNING, dag.getState());
+
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 1), VertexState.SUCCEEDED));
+ dag.handle(new DAGEvent(dagId, DAGEventType.DAG_KILL));
+
+ for (int i = 2; i < 5; ++i) {
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, i), VertexState.SUCCEEDED));
+ }
+ dispatcher.await();
+ Assert.assertEquals(DAGState.KILL_WAIT, dag.getState());
+
+ dag.handle(new DAGEventVertexCompleted(
+ new TezVertexID(dagId, 5), VertexState.KILLED));
+ dispatcher.await();
+ Assert.assertEquals(DAGState.KILLED, dag.getState());
+ Assert.assertEquals(5, dag.getSuccessfulVertices());
+ Assert.assertEquals(1, dagFinishEventHandler.dagFinishEvents);
+ }
+
+ @Test
+ public void testDiagnosticUpdates() {
+ // FIXME need to implement
+ }
+
+ @Test
+ public void testCounterUpdates() {
+ // FIXME need to implement
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8298190d/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index fe6ae14..f78a7b8 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -62,6 +62,8 @@ import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.VertexState;
import org.apache.tez.dag.app.dag.event.DAGEvent;
import org.apache.tez.dag.app.dag.event.DAGEventType;
+import org.apache.tez.dag.app.dag.event.TaskEvent;
+import org.apache.tez.dag.app.dag.event.TaskEventType;
import org.apache.tez.dag.app.dag.event.VertexEvent;
import org.apache.tez.dag.app.dag.event.VertexEventTaskAttemptCompleted;
import org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted;
@@ -115,7 +117,7 @@ public class TestVertexImpl {
public int abortCounter = 0;
private boolean throwError;
private boolean throwErrorOnAbort;
-
+
public CountingVertexOutputCommitter(boolean throwError,
boolean throwOnAbort) {
this.throwError = throwError;
@@ -125,7 +127,7 @@ public class TestVertexImpl {
public CountingVertexOutputCommitter() {
this(false, false);
}
-
+
@Override
public void init(VertexContext context) throws IOException {
++initCounter;
@@ -150,12 +152,26 @@ public class TestVertexImpl {
if (throwErrorOnAbort) {
throw new IOException("I can throwz exceptions in abort");
}
- }
+ }
}
-
+
+ private class TaskEventHandler implements EventHandler<TaskEvent> {
+ @Override
+ public void handle(TaskEvent event) {
+ }
+ }
+
private class DagEventDispatcher implements EventHandler<DAGEvent> {
+ public Map<DAGEventType, Integer> eventCount =
+ new HashMap<DAGEventType, Integer>();
+
@Override
public void handle(DAGEvent event) {
+ int count = 1;
+ if (eventCount.containsKey(event.getType())) {
+ count = eventCount.get(event.getType()) + 1;
+ }
+ eventCount.put(event.getType(), count);
}
}
@@ -164,7 +180,7 @@ public class TestVertexImpl {
public void handle(DAGHistoryEvent event) {
}
}
-
+
private class VertexEventDispatcher
implements EventHandler<VertexEvent> {
@@ -421,7 +437,7 @@ public class TestVertexImpl {
Map<Vertex, EdgeProperty> outVertices =
new HashMap<Vertex, EdgeProperty>();
-
+
for(String inEdgeId : vertexPlan.getInEdgeIdList()){
EdgePlan edgePlan = edgePlans.get(inEdgeId);
Vertex inVertex = this.vertices.get(edgePlan.getInputVertexName());
@@ -469,6 +485,7 @@ public class TestVertexImpl {
dispatcher.register(DAGEventType.class, dagEventDispatcher);
dispatcher.register(HistoryEventType.class,
new HistoryHandler());
+ dispatcher.register(TaskEventType.class, new TaskEventHandler());
dispatcher.init(conf);
dispatcher.start();
}
@@ -500,10 +517,10 @@ public class TestVertexImpl {
if (checkKillWait) {
Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
} else {
- Assert.assertEquals(VertexState.KILLED, v.getState());
+ Assert.assertEquals(VertexState.KILLED, v.getState());
}
}
-
+
private void startVertex(VertexImpl v,
boolean checkRunningState) {
Assert.assertEquals(VertexState.INITED, v.getState());
@@ -522,10 +539,10 @@ public class TestVertexImpl {
VertexImpl v3 = vertices.get("vertex3");
initVertex(v3);
-
+
Assert.assertEquals("x3.y3", v3.getProcessorName());
Assert.assertEquals("foo", v3.getJavaOpts());
-
+
Assert.assertEquals(2, v3.getInputSpecList().size());
Assert.assertEquals(2, v3.getInputVerticesCount());
Assert.assertEquals(2, v3.getOutputVerticesCount());
@@ -547,7 +564,7 @@ public class TestVertexImpl {
.getInputClassName())
|| "i3_v2".equals(v3.getInputSpecList().get(1)
.getInputClassName()));
-
+
Assert.assertTrue("vertex4".equals(v3.getOutputSpecList().get(0)
.getVertexName())
|| "vertex5".equals(v3.getOutputSpecList().get(0)
@@ -606,7 +623,7 @@ public class TestVertexImpl {
v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, v.getState());
-
+
v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, v.getState());
@@ -632,7 +649,7 @@ public class TestVertexImpl {
StringUtils.join(",", v.getDiagnostics()).toLowerCase();
Assert.assertTrue(diagnostics.contains("task failed " + t1.toString()));
}
-
+
@Test
public void testVertexWithNoTasks() {
// FIXME a vertex with no tasks should not be allowed
@@ -644,14 +661,14 @@ public class TestVertexImpl {
}
@Test
- public void testVertexKill() {
+ public void testVertexKillDiagnostics() {
VertexImpl v1 = vertices.get("vertex1");
killVertex(v1, false);
String diagnostics =
StringUtils.join(",", v1.getDiagnostics()).toLowerCase();
Assert.assertTrue(diagnostics.contains(
"vertex received kill in new state"));
-
+
VertexImpl v2 = vertices.get("vertex2");
initVertex(v2);
killVertex(v2, false);
@@ -672,30 +689,73 @@ public class TestVertexImpl {
}
@Test
- public void testKilledTasksHandling() {
+ public void testVertexKillPending() {
VertexImpl v = vertices.get("vertex2");
initVertex(v);
+ VertexImpl v3 = vertices.get("vertex3");
+ initVertex(v3);
+
startVertex(v);
- TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
- TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+ v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+ Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+
+ v.handle(new VertexEventTaskCompleted(
+ new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
- v.handle(new VertexEventTaskCompleted(t1, TaskState.KILLED));
+ v.handle(new VertexEventTaskCompleted(
+ new TezTaskID(v.getVertexId(), 1), TaskState.KILLED));
dispatcher.await();
- Assert.assertEquals(VertexState.RUNNING, v.getState());
+ Assert.assertEquals(VertexState.KILLED, v.getState());
+ }
+
+ @Test
+ @Ignore
+ public void testVertexKill() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ VertexImpl v3 = vertices.get("vertex3");
+ initVertex(v3);
- v.handle(new VertexEventTaskCompleted(t2, TaskState.KILLED));
+ startVertex(v);
+
+ v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+ Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+
+ v.handle(new VertexEventTaskCompleted(
+ new TezTaskID(v.getVertexId(), 0), TaskState.SUCCEEDED));
+ Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+
+ v.handle(new VertexEventTaskCompleted(
+ new TezTaskID(v.getVertexId(), 1), TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.KILLED, v.getState());
}
@Test
+ @Ignore
+ public void testKilledTasksHandling() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ startVertex(v);
+
+ TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+ TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.FAILED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.FAILED, v.getState());
+ Assert.assertEquals(TaskState.KILLED, v.getTask(t2).getState());
+ }
+
+ @Test
public void testVertexCommitterInit() {
VertexImpl v2 = vertices.get("vertex2");
initVertex(v2);
Assert.assertTrue(v2.getVertexOutputCommitter()
instanceof NullVertexOutputCommitter);
-
+
VertexImpl v6 = vertices.get("vertex6");
initVertex(v6);
Assert.assertTrue(v6.getVertexOutputCommitter()
@@ -708,13 +768,13 @@ public class TestVertexImpl {
initVertex(v2);
Assert.assertTrue(v2.getVertexScheduler()
instanceof ImmediateStartVertexScheduler);
-
+
VertexImpl v6 = vertices.get("vertex6");
initVertex(v6);
Assert.assertTrue(v6.getVertexScheduler()
instanceof BipartiteSlowStartVertexScheduler);
}
-
+
@Test
public void testVertexTaskFailure() {
VertexImpl v = vertices.get("vertex2");
@@ -730,7 +790,7 @@ public class TestVertexImpl {
v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, v.getState());
-
+
v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
dispatcher.await();
@@ -765,9 +825,9 @@ public class TestVertexImpl {
public void testDiagnostics() {
// FIXME need to test diagnostics in various cases
}
-
+
@Test
- public void testTaskAttemptCompletionEvents() {
+ public void testTaskAttemptCompletionEvents() {
// FIXME need to test handling of task attempt events
}
@@ -821,7 +881,7 @@ public class TestVertexImpl {
v4.handle(new VertexEventTaskAttemptCompleted(cEvt3));
v5.handle(new VertexEventTaskAttemptCompleted(cEvt4));
v5.handle(new VertexEventTaskAttemptCompleted(cEvt5));
- v5.handle(new VertexEventTaskAttemptCompleted(cEvt6));
+ v5.handle(new VertexEventTaskAttemptCompleted(cEvt6));
v4.handle(new VertexEventTaskCompleted(t1_v4, TaskState.SUCCEEDED));
v4.handle(new VertexEventTaskCompleted(t2_v4, TaskState.SUCCEEDED));
@@ -850,7 +910,9 @@ public class TestVertexImpl {
v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
-
+ Assert.assertEquals(1,
+ dagEventDispatcher.eventCount.get(
+ DAGEventType.DAG_VERTEX_COMPLETED).intValue());
}
@Test
@@ -861,7 +923,7 @@ public class TestVertexImpl {
CountingVertexOutputCommitter committer =
new CountingVertexOutputCommitter();
v.setVertexOutputCommitter(committer);
-
+
startVertex(v);
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
@@ -873,14 +935,14 @@ public class TestVertexImpl {
// v.handle(new VertexEventTaskReschedule(t1));
v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
- Assert.assertEquals(VertexState.RUNNING, v.getState());
+ Assert.assertEquals(VertexState.RUNNING, v.getState());
Assert.assertEquals(0, committer.commitCounter);
-
+
v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.await();
- Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(1, committer.commitCounter);
-
+
}
@Test
@@ -890,7 +952,7 @@ public class TestVertexImpl {
CountingVertexOutputCommitter committer =
new CountingVertexOutputCommitter();
v.setVertexOutputCommitter(committer);
-
+
startVertex(v);
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
@@ -900,23 +962,23 @@ public class TestVertexImpl {
v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
- Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(1, committer.commitCounter);
-
+
v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
- Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(1, committer.commitCounter);
Assert.assertEquals(0, committer.abortCounter);
Assert.assertEquals(0, committer.initCounter); // already done in init
Assert.assertEquals(0, committer.setupCounter); // already done in init
}
-
+
@Test
public void testCommitterInitAndSetup() {
// FIXME need to add a test for this
}
-
+
@Test
public void testTaskAttemptFetchFailureHandling() {
// FIXME needs testing
@@ -929,7 +991,7 @@ public class TestVertexImpl {
CountingVertexOutputCommitter committer =
new CountingVertexOutputCommitter(true, true);
v.setVertexOutputCommitter(committer);
-
+
startVertex(v);
TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
@@ -938,13 +1000,28 @@ public class TestVertexImpl {
v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
- Assert.assertEquals(VertexState.FAILED, v.getState());
+ Assert.assertEquals(VertexState.FAILED, v.getState());
Assert.assertEquals(1, committer.commitCounter);
-
+
// FIXME need to verify whether abort needs to be called if commit fails
Assert.assertEquals(0, committer.abortCounter);
Assert.assertEquals(0, committer.initCounter); // already done in init
- Assert.assertEquals(0, committer.setupCounter); // already done in init
+ Assert.assertEquals(0, committer.setupCounter); // already done in init
+ }
+
+ @Test
+ public void testHistoryEventGeneration() {
+ }
+
+ @Test
+ public void testInvalidEvent() {
+ VertexImpl v = vertices.get("vertex2");
+ v.handle(new VertexEvent(v.getVertexId(),
+ VertexEventType.V_START));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.ERROR, v.getState());
+ Assert.assertEquals(1,
+ dagEventDispatcher.eventCount.get(
+ DAGEventType.INTERNAL_ERROR).intValue());
}
-
}
[23/50] [abbrv] git commit: TEZ-152. Add a way for MR tasks to figure
out which stage they belong to. (sseth)
Posted by ss...@apache.org.
TEZ-152. Add a way for MR tasks to figure out which stage they belong
to. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/7d401d2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/7d401d2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/7d401d2c
Branch: refs/heads/master
Commit: 7d401d2cd0af2c68ff2a060849b71e1b0796c27d
Parents: ef37823
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue May 21 18:51:32 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue May 21 18:51:32 2013 -0700
----------------------------------------------------------------------
.../apache/tez/mapreduce/hadoop/MRJobConfig.java | 2 +
.../mapreduce/hadoop/MultiStageMRConfigUtil.java | 17 +++++++++++++++
.../apache/tez/mapreduce/task/MRRuntimeTask.java | 1 +
3 files changed, 20 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7d401d2c/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
index 7e650b1..a42597e 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
@@ -649,6 +649,8 @@ public interface MRJobConfig {
public static final String MRR_VERTEX_PREFIX =
"mrr.vertex.";
+ public static final String VERTEX_NAME = "mapreduce.task.vertex.name";
+
// Stage specific properties
// Format of each property is mapred.ireducer.stage.<stage-num>.<suffix>
// where suffix is one of MRR_INTERMEDIATE_STAGE_* fields defined below.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7d401d2c/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java
index b4b34e0..fd3dd78 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java
@@ -96,14 +96,31 @@ public class MultiStageMRConfigUtil {
public static String getInitialMapVertexName() {
return INITIAL_MAP_VERTEX_NAME;
}
+
+ public boolean isInitialMapVertex(String vertexName) {
+ return vertexName.equals(INITIAL_MAP_VERTEX_NAME);
+ }
public static String getFinalReduceVertexName() {
return FINAL_REDUCE_VERTEX_NAME;
}
+ public boolean isFinalReduceVertex(String vertexName) {
+ return vertexName.equals(FINAL_REDUCE_VERTEX_NAME);
+ }
+
public static String getIntermediateStageVertexName(int stageNum) {
return INTERMEDIATE_TASK_VERTEX_NAME_PREFIX + stageNum;
}
+
+ public static int getIntermediateStageNum(String vertexName) {
+ if (vertexName.matches(INTERMEDIATE_TASK_VERTEX_NAME_PREFIX + "\\d+")) {
+ return Integer.parseInt(vertexName
+ .substring(INTERMEDIATE_TASK_VERTEX_NAME_PREFIX.length()));
+ } else {
+ return -1;
+ }
+ }
// Returns config settings specific to named vertex
public static Configuration getBasicConfForVertex(Configuration baseConf,
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7d401d2c/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index e5dfd66..34aaa1b 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -97,6 +97,7 @@ public class MRRuntimeTask extends RuntimeTask {
// submit configuration parameters to the AM and effectively tasks via RPC.
final JobConf job = new JobConf(taskConf);
+ job.set(MRJobConfig.VERTEX_NAME, taskContext.getVertexName());
MRTask mrTask = (MRTask) getProcessor();
this.mrTask = mrTask;
[32/50] [abbrv] git commit: TEZ-27. Create tests for DAG Scheduler
(bikas)
Posted by ss...@apache.org.
TEZ-27. Create tests for DAG Scheduler (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/82ee45e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/82ee45e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/82ee45e8
Branch: refs/heads/master
Commit: 82ee45e8d3d7f0168b49c534384217305ab81a6d
Parents: 3bb954b
Author: Bikas Saha <bi...@apache.org>
Authored: Tue May 28 19:20:30 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue May 28 19:20:30 2013 -0700
----------------------------------------------------------------------
.../org/apache/tez/dag/app/dag/TaskAttempt.java | 5 +
.../dag/app/dag/event/DAGEventSchedulerUpdate.java | 2 +-
.../org/apache/tez/dag/app/dag/impl/DAGImpl.java | 30 ++-
.../tez/dag/app/dag/impl/DAGSchedulerMRR.java | 87 +++----
.../dag/app/dag/impl/DAGSchedulerNaturalOrder.java | 2 +-
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 18 ++-
.../tez/dag/app/rm/TaskSchedulerEventHandler.java | 2 +-
.../tez/dag/app/dag/impl/TestDAGScheduler.java | 188 +++++++++++++++
8 files changed, 277 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
index 712f968..2628f0c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
@@ -27,13 +27,18 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.oldrecords.TaskAttemptReport;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
+import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezVertexID;
/**
* Read only view of TaskAttempt.
*/
public interface TaskAttempt {
TezTaskAttemptID getID();
+ TezVertexID getVertexID();
+ TezDAGID getDAGID();
+
TaskAttemptReport getReport();
List<String> getDiagnostics();
TezCounters getCounters();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java
index ff11e99..99b872e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java
@@ -30,7 +30,7 @@ public class DAGEventSchedulerUpdate extends DAGEvent {
private final UpdateType updateType;
public DAGEventSchedulerUpdate(UpdateType updateType, TaskAttempt attempt) {
- super(attempt.getID().getTaskID().getVertexID().getDAGId(),
+ super(attempt.getDAGID(),
DAGEventType.DAG_SCHEDULER_UPDATE);
this.attempt = attempt;
this.updateType = updateType;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index cc6699d..3681c85 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -869,9 +869,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
for (Vertex v : dag.vertices.values()) {
parseVertexEdges(dag, edgePlans, v);
}
-
- dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
- //dag.dagScheduler = new DAGSchedulerMRR(dag, dag.eventHandler);
+
+ assignDAGScheduler(dag);
// TODO Metrics
//dag.metrics.endPreparingJob(dag);
@@ -887,6 +886,31 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
return dag.finished(DAGState.FAILED);
}
}
+
+ private void assignDAGScheduler(DAGImpl dag) {
+ boolean isMRR = true;
+ for(Vertex vertex : dag.vertices.values()) {
+ Map<Vertex, EdgeProperty> outVertices = vertex.getOutputVertices();
+ if(outVertices == null || outVertices.isEmpty()) {
+ continue;
+ }
+ if(outVertices.size() > 1 ||
+ outVertices.values().iterator().next().getConnectionPattern() !=
+ EdgeProperty.ConnectionPattern.BIPARTITE) {
+ // more than 1 output OR single output is not bipartite
+ isMRR = false;
+ break;
+ }
+ }
+
+ if(isMRR) {
+ LOG.info("Using MRR dag scheduler");
+ dag.dagScheduler = new DAGSchedulerMRR(dag, dag.eventHandler);
+ } else {
+ LOG.info("Using Natural order dag scheduler");
+ dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
+ }
+ }
private VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
TezVertexID vertexId = TezBuilderUtils.newVertexID(dag.getID(), vId);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
index 12fdb10..d404155 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerMRR.java
@@ -48,7 +48,7 @@ public class DAGSchedulerMRR implements DAGScheduler {
@Override
public void vertexCompleted(Vertex vertex) {
- if(currentPartitioner!= null) {
+ if(currentPartitioner != null) {
if(vertex != currentPartitioner) {
String message = vertex.getVertexId() + " finished. Expecting "
+ currentPartitioner + " to finish.";
@@ -57,68 +57,59 @@ public class DAGSchedulerMRR implements DAGScheduler {
}
LOG.info("Current partitioner " + currentPartitioner.getVertexId()
+ " is completed. "
- + (currentShuffler!=null?currentShuffler.getVertexId():"null")
- + " is new partitioner");
+ + (currentShuffler!=null ?
+ currentShuffler.getVertexId() + " is new partitioner":
+ "No current shuffler to replace the partitioner"));
currentPartitioner = currentShuffler;
- currentShuffler = null;
- } else {
- if(vertex != currentShuffler) {
- String message = vertex.getVertexId() + " finished. Expecting "
- + currentShuffler.getVertexId() + " to finish";
- LOG.fatal(message);
- throw new TezException(message);
- }
+ currentShuffler = null;
}
}
@Override
public void scheduleTask(DAGEventSchedulerUpdate event) {
TaskAttempt attempt = event.getAttempt();
- Vertex vertex = dag.getVertex(attempt.getID().getTaskID().getVertexID());
+ Vertex vertex = dag.getVertex(attempt.getVertexID());
int vertexDistanceFromRoot = vertex.getDistanceFromRoot();
- if(vertexDistanceFromRoot == 0) {
+ boolean reOrderPriority = false;
+
+ if(currentPartitioner == null) {
+ // no partitioner. so set it.
currentPartitioner = vertex;
- LOG.info(vertex.getVertexId() + " is first partitioner");
+ currentShufflerDepth = vertexDistanceFromRoot;
+ LOG.info(vertex.getVertexId() + " is new partitioner at depth "
+ + vertexDistanceFromRoot);
+ } else if (currentShuffler == null &&
+ vertexDistanceFromRoot > currentShufflerDepth) {
+ // vertex not a partitioner. no shuffler set. has more depth than current
+ // shuffler. this must be the new shuffler.
+ currentShuffler = vertex;
+ currentShufflerDepth = vertexDistanceFromRoot;
+ LOG.info(vertex.getVertexId() + " is new shuffler at depth "
+ + currentShufflerDepth);
}
- if(vertexDistanceFromRoot > currentShufflerDepth) {
- if(currentShuffler == null) {
- currentShuffler = vertex;
- currentShufflerDepth = vertexDistanceFromRoot;
- LOG.info(currentShuffler.getVertexId() + " is new shuffler at depth " +
- currentShufflerDepth);
- } else {
- if(currentShufflerDepth+1 == vertexDistanceFromRoot &&
- currentPartitioner == null
- ) {
- currentPartitioner = currentShuffler;
- currentShuffler = vertex;
- currentShufflerDepth = vertexDistanceFromRoot;
- LOG.info("Shuffler " + currentPartitioner.getVertexId() +
- " becomes partitioner as new shuffler " +
- currentShuffler.getVertexId() + " has started at depth " +
- currentShufflerDepth);
- } else {
- String message = vertex.getVertexId()
- + " has scheduled tasks at depth " + vertexDistanceFromRoot
- + " greater than depth " + currentShufflerDepth
- + " of current shuffler " + currentShuffler.getVertexId()
- + ". Unexpected.";
- LOG.fatal(message);
- throw new TezException(message);
- }
- }
+
+ if(currentShuffler == vertex) {
+ // current shuffler vertex. assign special priority
+ reOrderPriority = true;
}
+
+ // sanity check
+ if(currentPartitioner != vertex && currentShuffler != vertex) {
+ String message = vertex.getVertexId() + " is neither the "
+ + " current partitioner: " + currentPartitioner.getVertexId()
+ + " nor the current shuffler: " + currentShuffler.getVertexId();
+ LOG.fatal(message);
+ throw new TezException(message);
+ }
// natural priority. Handles failures and retries.
int priority = (vertexDistanceFromRoot + 1) * 3;
- if(currentShuffler == vertex) {
- if(currentPartitioner != null) {
- // special priority for current reducers while current partitioners are
- // still running. Schedule at priority one higher than natural priority
- // of previous vertex.
- priority -= 4; // this == (partitionerDepth+1)*3 - 1
- }
+ if(reOrderPriority) {
+ // special priority for current reducers while current partitioners are
+ // still running. Schedule at priority one higher than natural priority
+ // of previous vertex.
+ priority -= 4; // this == (partitionerDepth+1)*3 - 1
} else {
if(attempt.getIsRescheduled()) {
// higher priority for retries of failed attempts. Only makes sense in
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
index 607e5af..9e8729e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java
@@ -50,7 +50,7 @@ public class DAGSchedulerNaturalOrder implements DAGScheduler {
@Override
public void scheduleTask(DAGEventSchedulerUpdate event) {
TaskAttempt attempt = event.getAttempt();
- Vertex vertex = dag.getVertex(attempt.getID().getTaskID().getVertexID());
+ Vertex vertex = dag.getVertex(attempt.getVertexID());
int vertexDistanceFromRoot = vertex.getDistanceFromRoot();
// natural priority. Handles failures and retries.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 5d4242d..5225ecf 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -90,8 +90,10 @@ import org.apache.tez.dag.app.rm.AMSchedulerEventTALaunchRequest;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.events.TaskAttemptFinishedEvent;
import org.apache.tez.dag.history.events.TaskAttemptStartedEvent;
+import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.dag.utils.TezBuilderUtils;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
@@ -290,6 +292,16 @@ public class TaskAttemptImpl implements TaskAttempt,
return attemptId;
}
+ @Override
+ public TezVertexID getVertexID() {
+ return attemptId.getTaskID().getVertexID();
+ }
+
+ @Override
+ public TezDAGID getDAGID() {
+ return getVertexID().getDAGId();
+ }
+
TezTaskContext createRemoteTask() {
Vertex vertex = getTask().getVertex();
DAG dag = vertex.getDAG();
@@ -601,7 +613,7 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptImpl ta) {
DAGEventCounterUpdate jce =
new DAGEventCounterUpdate(
- ta.getID().getTaskID().getVertexID().getDAGId()
+ ta.getDAGID()
);
jce.addCounterUpdate(DAGCounter.TOTAL_LAUNCHED_TASKS, 1);
return jce;
@@ -611,7 +623,7 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptImpl ta) {
DAGEventCounterUpdate jce =
new DAGEventCounterUpdate(
- ta.getID().getTaskID().getVertexID().getDAGId()
+ ta.getDAGID()
);
long slotMillis = computeSlotMillis(ta);
@@ -624,7 +636,7 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptStateInternal taState) {
DAGEventCounterUpdate jce =
new DAGEventCounterUpdate(
- taskAttempt.getID().getTaskID().getVertexID().getDAGId());
+ taskAttempt.getDAGID());
long slotMillisIncrement = computeSlotMillis(taskAttempt);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index c61cbcc..8b98c68 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -461,7 +461,7 @@ public class TaskSchedulerEventHandler extends AbstractService
sendEvent(new AMContainerEventLaunchRequest(
containerId,
- taskAttempt.getID().getTaskID().getVertexID(),
+ taskAttempt.getVertexID(),
event.getJobToken(),
// TODO getConf from AMSchedulerEventTALaunchRequest
event.getCredentials(), false, event.getConf(),
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/82ee45e8/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
new file mode 100644
index 0000000..988a116
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.DAGScheduler;
+import org.apache.tez.dag.app.dag.TaskAttempt;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdate;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.mockito.Mockito.*;
+
+public class TestDAGScheduler {
+
+ class MockEventHandler implements EventHandler<TaskAttemptEventSchedule> {
+ TaskAttemptEventSchedule event;
+ @Override
+ public void handle(TaskAttemptEventSchedule event) {
+ this.event = event;
+ }
+
+ }
+
+ MockEventHandler mockEventHandler = new MockEventHandler();
+
+ //@Test(timeout=10000)
+ public void testDAGSchedulerNaturalOrder() {
+ DAG mockDag = mock(DAG.class);
+ Vertex mockVertex = mock(Vertex.class);
+ TaskAttempt mockAttempt = mock(TaskAttempt.class);
+ when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex);
+ when(mockVertex.getDistanceFromRoot()).thenReturn(0).thenReturn(1)
+ .thenReturn(2);
+ when(mockAttempt.getIsRescheduled()).thenReturn(false);
+
+ DAGEventSchedulerUpdate event = new DAGEventSchedulerUpdate(
+ DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt);
+
+ DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag,
+ mockEventHandler);
+ scheduler.scheduleTask(event);
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 2);
+ scheduler.scheduleTask(event);
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 4);
+ scheduler.scheduleTask(event);
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+
+ when(mockAttempt.getIsRescheduled()).thenReturn(true);
+ scheduler.scheduleTask(event);
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 5);
+ }
+
+ @Test//(timeout=10000)
+ public void testDAGSchedulerMRR() {
+ DAG mockDag = mock(DAG.class);
+ TezDAGID dagId = new TezDAGID("1", 1, 1);
+
+ Vertex mockVertex1 = mock(Vertex.class);
+ TezVertexID mockVertexId1 = new TezVertexID(dagId, 1);
+ when(mockVertex1.getVertexId()).thenReturn(mockVertexId1);
+ when(mockVertex1.getDistanceFromRoot()).thenReturn(0);
+ TaskAttempt mockAttempt1 = mock(TaskAttempt.class);
+ when(mockAttempt1.getVertexID()).thenReturn(mockVertexId1);
+ when(mockAttempt1.getIsRescheduled()).thenReturn(false);
+ when(mockDag.getVertex(mockVertexId1)).thenReturn(mockVertex1);
+
+ Vertex mockVertex2 = mock(Vertex.class);
+ TezVertexID mockVertexId2 = new TezVertexID(dagId, 2);
+ when(mockVertex2.getVertexId()).thenReturn(mockVertexId2);
+ when(mockVertex2.getDistanceFromRoot()).thenReturn(1);
+ TaskAttempt mockAttempt2 = mock(TaskAttempt.class);
+ when(mockAttempt2.getVertexID()).thenReturn(mockVertexId2);
+ when(mockAttempt2.getIsRescheduled()).thenReturn(false);
+ when(mockDag.getVertex(mockVertexId2)).thenReturn(mockVertex2);
+ TaskAttempt mockAttempt2f = mock(TaskAttempt.class);
+ when(mockAttempt2f.getVertexID()).thenReturn(mockVertexId2);
+ when(mockAttempt2f.getIsRescheduled()).thenReturn(true);
+
+ Vertex mockVertex3 = mock(Vertex.class);
+ TezVertexID mockVertexId3 = new TezVertexID(dagId, 3);
+ when(mockVertex3.getVertexId()).thenReturn(mockVertexId3);
+ when(mockVertex3.getDistanceFromRoot()).thenReturn(2);
+ TaskAttempt mockAttempt3 = mock(TaskAttempt.class);
+ when(mockAttempt3.getVertexID()).thenReturn(mockVertexId3);
+ when(mockAttempt3.getIsRescheduled()).thenReturn(false);
+ when(mockDag.getVertex(mockVertexId3)).thenReturn(mockVertex3);
+
+ DAGEventSchedulerUpdate mockEvent1 = mock(DAGEventSchedulerUpdate.class);
+ when(mockEvent1.getAttempt()).thenReturn(mockAttempt1);
+ DAGEventSchedulerUpdate mockEvent2 = mock(DAGEventSchedulerUpdate.class);
+ when(mockEvent2.getAttempt()).thenReturn(mockAttempt2);
+ DAGEventSchedulerUpdate mockEvent2f = mock(DAGEventSchedulerUpdate.class);
+ when(mockEvent2f.getAttempt()).thenReturn(mockAttempt2f);
+ DAGEventSchedulerUpdate mockEvent3 = mock(DAGEventSchedulerUpdate.class);
+ when(mockEvent3.getAttempt()).thenReturn(mockAttempt3);
+ DAGScheduler scheduler = new DAGSchedulerMRR(mockDag, mockEventHandler);
+
+ // M starts. M completes. R1 starts. R1 completes. R2 starts. R2 completes
+ scheduler.scheduleTask(mockEvent1); // M starts
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+ scheduler.scheduleTask(mockEvent1); // M runs another
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+ scheduler.vertexCompleted(mockVertex1); // M completes
+ scheduler.scheduleTask(mockEvent2); // R1 starts
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+ scheduler.scheduleTask(mockEvent2); // R1 runs another
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+ scheduler.scheduleTask(mockEvent2f); // R1 runs retry. Retry priority
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 4);
+ scheduler.vertexCompleted(mockVertex2); // R1 completes
+ scheduler.scheduleTask(mockEvent3); // R2 starts
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 9);
+ scheduler.scheduleTask(mockEvent3); // R2 runs another
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 9);
+ scheduler.vertexCompleted(mockVertex3); // R2 completes
+
+ // M starts. R1 starts. M completes. R2 starts. R1 completes. R2 completes
+ scheduler.scheduleTask(mockEvent1); // M starts
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+ scheduler.scheduleTask(mockEvent2); // R1 starts. Reordered priority
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 2);
+ scheduler.scheduleTask(mockEvent1); // M runs another
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+ scheduler.scheduleTask(mockEvent2); // R1 runs another. Reordered priority
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 2);
+ scheduler.scheduleTask(mockEvent2f); // R1 runs retry. Reordered priority
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 2);
+ scheduler.vertexCompleted(mockVertex1); // M completes
+ scheduler.scheduleTask(mockEvent3); // R2 starts. Reordered priority
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 5);
+ scheduler.scheduleTask(mockEvent2); // R1 runs another. Normal priority
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+ scheduler.scheduleTask(mockEvent2f); // R1 runs retry. Retry priority
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 4);
+ scheduler.scheduleTask(mockEvent3); // R2 runs another. Reordered priority
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 5);
+ scheduler.vertexCompleted(mockVertex2); // R1 completes
+ scheduler.vertexCompleted(mockVertex3); // R2 completes
+
+ // M starts. M completes. R1 starts. R2 starts. R1 completes. R2 completes
+ scheduler.scheduleTask(mockEvent1); // M starts
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+ scheduler.vertexCompleted(mockVertex1); // M completes
+ scheduler.scheduleTask(mockEvent2); // R1 starts
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+ scheduler.scheduleTask(mockEvent3); // R2 starts. Reordered priority
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 5);
+ scheduler.scheduleTask(mockEvent2); // R1 runs another
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+ scheduler.vertexCompleted(mockVertex2); // R1 completes
+ scheduler.vertexCompleted(mockVertex3); // R2 completes
+
+ // M starts. R1 starts. M completes. R1 completes. R2 starts. R2 completes
+ scheduler.scheduleTask(mockEvent1); // M starts
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 3);
+ scheduler.scheduleTask(mockEvent2); // R1 starts. Reordered priority
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 2);
+ scheduler.vertexCompleted(mockVertex1); // M completes
+ scheduler.scheduleTask(mockEvent2); // R1 starts. Normal priority
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 6);
+ scheduler.vertexCompleted(mockVertex2); // R1 completes
+ scheduler.scheduleTask(mockEvent3); // R2 starts
+ Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 9);
+ scheduler.vertexCompleted(mockVertex3); // R2 completes
+ }
+}
[07/50] [abbrv] TEZ-123. Cleanup package dependencies. (bikas)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
index f9e8eb7..6c7bb0f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
@@ -29,9 +29,9 @@ import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.VertexScheduler;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
/**
* Starts scheduling tasks when number of completed source tasks crosses
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index cc3170e..02e91b4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -58,8 +58,8 @@ import org.apache.tez.dag.api.DAGProtos.VertexPlan;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint;
-import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
+import org.apache.tez.dag.api.impl.DAGStatus;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
@@ -81,12 +81,12 @@ import org.apache.tez.dag.app.dag.event.VertexEventType;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.events.DAGFinishedEvent;
import org.apache.tez.dag.history.events.DAGStartedEvent;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.dag.utils.DAGApps;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
import org.apache.tez.engine.common.security.JobTokenSecretManager;
import org.apache.tez.engine.common.security.TokenCache;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezVertexID;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
/** Implementation of Job interface. Maintains the state machines of Job.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexScheduler.java
index 1d13aa4..bd39115 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexScheduler.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.app.dag.impl;
import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.VertexScheduler;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
/**
* Starts all tasks immediately on vertex start
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index c907e48..2564ffa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -91,9 +91,9 @@ import org.apache.tez.dag.app.rm.AMSchedulerEventTALaunchRequest;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.events.TaskAttemptFinishedEvent;
import org.apache.tez.dag.history.events.TaskAttemptStartedEvent;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index cf49dbd..4d0c4f6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -67,11 +67,11 @@ import org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.events.TaskFinishedEvent;
import org.apache.tez.dag.history.events.TaskStartedEvent;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index b9e911c..3f3ac49 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -62,11 +62,11 @@ import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.client.VertexStatus;
-import org.apache.tez.dag.api.client.VertexStatus.State;
import org.apache.tez.dag.api.impl.NullVertexOutputCommitter;
import org.apache.tez.dag.api.impl.VertexContext;
import org.apache.tez.dag.api.impl.VertexOutputCommitter;
+import org.apache.tez.dag.api.impl.VertexStatus;
+import org.apache.tez.dag.api.impl.VertexStatus.State;
import org.apache.tez.dag.api.records.TaskState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
@@ -94,12 +94,12 @@ import org.apache.tez.dag.app.dag.event.VertexEventType;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.events.VertexFinishedEvent;
import org.apache.tez.dag.history.events.VertexStartedEvent;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezDAGID;
import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
/** Implementation of Vertex interface. Maintains the state machines of Vertex.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
index 3a32662..fddd204 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.app.rm;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.tez.dag.api.records.TaskAttemptState;
import org.apache.tez.dag.app.dag.TaskAttempt;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class AMSchedulerEventTAEnded extends AMSchedulerEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
index 04fd740..b595c2b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java
@@ -27,8 +27,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.tez.common.TezTaskContext;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.app.dag.TaskAttempt;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezTaskAttemptID;
public class AMSchedulerEventTALaunchRequest extends AMSchedulerEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
index 3637ded..3c00620 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java
@@ -23,7 +23,7 @@ import java.util.List;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public interface AMContainer extends EventHandler<AMContainerEvent>{
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
index 7bc0d7e..0fe5b3b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java
@@ -19,7 +19,7 @@ package org.apache.tez.dag.app.rm.container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.tez.common.TezTaskContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class AMContainerEventAssignTA extends AMContainerEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
index e672052..4e199da 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java
@@ -25,9 +25,9 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezVertexID;
public class AMContainerEventLaunchRequest extends AMContainerEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java
index 1deb9f4..e0c8fe7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.app.rm.container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class AMContainerEventTASucceeded extends AMContainerEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index c02ed7a..5eca572 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -49,11 +49,11 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.dag.utils.TezEngineChildJVM;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
import org.apache.tez.engine.common.security.TokenCache;
import org.apache.tez.engine.common.shuffle.server.ShuffleHandler;
-import org.apache.tez.engine.records.TezVertexID;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 474af31..1089013 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -56,7 +56,7 @@ import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
@SuppressWarnings("rawtypes")
public class AMContainerImpl implements AMContainer {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
index 9173713..cc2b7f7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java
@@ -19,7 +19,7 @@ package org.apache.tez.dag.app.rm.node;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class AMNodeEventTaskAttemptEnded extends AMNodeEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java
index 6599efa..b07d594 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java
@@ -19,7 +19,7 @@ package org.apache.tez.dag.app.rm.node;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class AMNodeEventTaskAttemptSucceeded extends AMNodeEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
index ccd148c..04c104a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
@@ -47,9 +47,9 @@ import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
import org.apache.tez.dag.app.dag.event.TaskEvent;
import org.apache.tez.dag.app.dag.event.TaskEventType;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
// FIXME does not handle multiple vertices
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java
index 83196b0..558f78c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
/**
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
index 4ef9991..aa95f37 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
@@ -28,8 +28,8 @@ import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
public class LegacyTaskRuntimeEstimator extends StartEndTimesBase {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/NullTaskRuntimesEngine.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/NullTaskRuntimesEngine.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/NullTaskRuntimesEngine.java
index e49c9fb..9907bc6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/NullTaskRuntimesEngine.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/NullTaskRuntimesEngine.java
@@ -21,8 +21,8 @@ package org.apache.tez.dag.app.speculate;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
/*
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/SpeculatorEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/SpeculatorEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/SpeculatorEvent.java
index 08697ee..c8f400b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/SpeculatorEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/SpeculatorEvent.java
@@ -20,9 +20,9 @@ package org.apache.tez.dag.app.speculate;
import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
public class SpeculatorEvent extends AbstractEvent<Speculator.EventType> {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
index 317d9c1..191d0b9 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
@@ -33,9 +33,9 @@ import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
abstract class StartEndTimesBase<V> implements TaskRuntimeEstimator {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskRuntimeEstimator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskRuntimeEstimator.java
index 81aa9c9..965d965 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskRuntimeEstimator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskRuntimeEstimator.java
@@ -21,8 +21,8 @@ package org.apache.tez.dag.app.speculate;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttemptStatus;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskSpeculationPredicate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskSpeculationPredicate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskSpeculationPredicate.java
index 6516bac..33ea09f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskSpeculationPredicate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/TaskSpeculationPredicate.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.app.speculate;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.Task;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
public class TaskSpeculationPredicate {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanupEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanupEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanupEvent.java
index 1839f53..9b6cc15 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanupEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/taskclean/TaskCleanupEvent.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.app.taskclean;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
/**
* This class encapsulates task cleanup event.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java
index ba9d258..d3b8a91 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.history;
import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
public class DAGHistoryEvent extends AbstractEvent<HistoryEventType>{
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
index 547a09a..d1b87b1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
@@ -18,11 +18,11 @@
package org.apache.tez.dag.history.events;
-import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.impl.DAGStatus;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.DAGFinished;
import org.apache.tez.dag.history.avro.HistoryEventType;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
public class DAGFinishedEvent implements HistoryEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java
index 104e949..719538c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.history.events;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.DAGStarted;
import org.apache.tez.dag.history.avro.HistoryEventType;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
public class DAGStartedEvent implements HistoryEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
index 7b5f3a1..4a2917e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
@@ -22,7 +22,7 @@ import org.apache.tez.dag.api.records.TaskAttemptState;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
import org.apache.tez.dag.history.avro.TaskAttemptFinished;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptFinishedEvent implements HistoryEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
index ea3e71f..97c9f2f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
import org.apache.tez.dag.history.avro.TaskAttemptStarted;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptStartedEvent implements HistoryEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
index 270e50a..165dc20 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
@@ -22,7 +22,7 @@ import org.apache.tez.dag.api.records.TaskState;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
import org.apache.tez.dag.history.avro.TaskFinished;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
public class TaskFinishedEvent implements HistoryEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java
index 6ed7868..964cc79 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.history.events;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
import org.apache.tez.dag.history.avro.TaskStarted;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
public class TaskStartedEvent implements HistoryEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
index b6d4d0b..b02be1e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
@@ -18,11 +18,11 @@
package org.apache.tez.dag.history.events;
-import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.impl.VertexStatus;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
import org.apache.tez.dag.history.avro.VertexFinished;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezVertexID;
public class VertexFinishedEvent implements HistoryEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java
index 98367dd..a4fa20e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.history.events;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
import org.apache.tez.dag.history.avro.VertexStarted;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezVertexID;
public class VertexStartedEvent implements HistoryEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
index fee0846..727ded5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
@@ -30,7 +30,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.app.rm.container.AMContainerHelpers;
+import org.apache.tez.dag.records.TezVertexID;
public class TezEngineChildJVM {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index 12ddcf0..a18bd3b 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -66,11 +66,11 @@ import org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
import org.apache.tez.dag.app.rm.container.AMContainerMap;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
import org.junit.Test;
@SuppressWarnings({ "unchecked", "rawtypes" })
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 13a19b9..516ea5e 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -52,11 +52,11 @@ import org.apache.tez.dag.app.dag.TaskStateInternal;
import org.apache.tez.dag.app.dag.event.TaskEvent;
import org.apache.tez.dag.app.dag.event.TaskEventTAUpdate;
import org.apache.tez.dag.app.dag.event.TaskEventType;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-engine-api/pom.xml b/tez-engine-api/pom.xml
index 7a747d2..2cc7276 100644
--- a/tez-engine-api/pom.xml
+++ b/tez-engine-api/pom.xml
@@ -25,6 +25,10 @@
<artifactId>tez-engine-api</artifactId>
<dependencies>
<dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-common</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/api/Master.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/api/Master.java b/tez-engine-api/src/main/java/org/apache/tez/engine/api/Master.java
index 3358a03..f3add9a 100644
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/api/Master.java
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/api/Master.java
@@ -19,7 +19,7 @@ package org.apache.tez.engine.api;
import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.ipc.VersionedProtocol;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
/**
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDAGID.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDAGID.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDAGID.java
deleted file mode 100644
index 1066409..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDAGID.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.records;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.text.NumberFormat;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.BuilderUtils;
-
-
-/**
- * TezDAGID represents the immutable and unique identifier for
- * a Tez DAG.
- *
- * TezDAGID consists of 2 parts. The first part is the {@link ApplicationId},
- * that is the YARN Application ID that this DAG belongs to. The second part is
- * the DAG number.
- *
- * @see ApplicationId
- */
-public class TezDAGID extends TezID {
-
- public static final String DAG = "dag";
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
- static {
- idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(6);
- }
-
- private ApplicationId applicationId;
-
- public TezDAGID() {
- }
-
- /**
- * Constructs a DAGID object from given {@link ApplicationId}.
- * @param applicationId Application that this dag belongs to
- * @param id the dag number
- */
- public TezDAGID(ApplicationId applicationId, int id) {
- super(id);
- if(applicationId == null) {
- throw new IllegalArgumentException("applicationId cannot be null");
- }
- this.applicationId = applicationId;
- }
-
- /**
- * Constructs a DAGID object from given parts.
- * @param yarnRMIdentifier YARN RM identifier
- * @param applicationId application number
- * @param id the dag number
- */
- public TezDAGID(String yarnRMIdentifier, int appId, int id) {
- this(BuilderUtils.newApplicationId(Long.valueOf(yarnRMIdentifier),
- appId), id);
- }
-
- /** Returns the {@link ApplicationId} object that this dag belongs to */
- public ApplicationId getApplicationId() {
- return applicationId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!super.equals(o))
- return false;
-
- TezDAGID that = (TezDAGID)o;
- return this.applicationId.equals(that.applicationId);
- }
-
- /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
- @Override
- public int compareTo(TezID o) {
- TezDAGID that = (TezDAGID)o;
- return this.applicationId.compareTo(that.applicationId);
- }
-
- @Override
- public String toString() {
- return appendTo(new StringBuilder(DAG)).toString();
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- applicationId = BuilderUtils.newApplicationId(in.readLong(), in.readInt());
- super.readFields(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeLong(applicationId.getClusterTimestamp());
- out.writeInt(applicationId.getId());
- super.write(out);
- }
-
- /**
- * Add the unique string to the given builder.
- * @param builder the builder to append to
- * @return the builder that was passed in
- */
- protected StringBuilder appendTo(StringBuilder builder) {
- return builder.append(SEPARATOR).
- append(applicationId.getClusterTimestamp()).
- append(SEPARATOR).
- append(applicationId.getId()).
- append(SEPARATOR).
- append(idFormat.format(id));
- }
-
- @Override
- public int hashCode() {
- return applicationId.hashCode() * 524287 + id;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDependentTaskCompletionEvent.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDependentTaskCompletionEvent.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDependentTaskCompletionEvent.java
index 91ce53c..cba752e 100644
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDependentTaskCompletionEvent.java
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezDependentTaskCompletionEvent.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.tez.dag.records.TezTaskAttemptID;
/**
* This is used to track task completion events on
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezID.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezID.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezID.java
deleted file mode 100644
index 13ea05a..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezID.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.records;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.WritableComparable;
-
-/**
- * A general identifier, which internally stores the id
- * as an integer. This is the super class of {@link TezJobID},
- * {@link TezTaskID} and {@link TezTaskAttemptID}.
- *
- * @see TezTaskID
- * @see TezTaskAttemptID
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public abstract class TezID implements WritableComparable<TezID> {
- public static final char SEPARATOR = '_';
- protected int id;
-
- /** constructs an ID object from the given int */
- public TezID(int id) {
- this.id = id;
- }
-
- protected TezID() {
- }
-
- /** returns the int which represents the identifier */
- public int getId() {
- return id;
- }
-
- @Override
- public String toString() {
- return String.valueOf(id);
- }
-
- @Override
- public int hashCode() {
- return id;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if(o == null)
- return false;
- if (o.getClass() == this.getClass()) {
- TezID that = (TezID) o;
- return this.id == that.id;
- }
- else
- return false;
- }
-
- /** Compare IDs by associated numbers */
- public int compareTo(TezID that) {
- return this.id - that.id;
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- this.id = in.readInt();
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(id);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskAttemptID.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskAttemptID.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskAttemptID.java
deleted file mode 100644
index e59c5bf..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskAttemptID.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.records;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * TezTaskAttemptID represents the immutable and unique identifier for
- * a task attempt. Each task attempt is one particular instance of a Tez Task
- * identified by its TezTaskID.
- *
- * TezTaskAttemptID consists of 2 parts. First part is the
- * {@link TezTaskID}, that this TaskAttemptID belongs to.
- * Second part is the task attempt number. <br>
- * <p>
- * Applications should never construct or parse TaskAttemptID strings
- * , but rather use appropriate constructors or {@link #forName(String)}
- * method.
- *
- * @see TezTaskID
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class TezTaskAttemptID extends TezID {
- public static final String ATTEMPT = "attempt";
- private TezTaskID taskId;
-
- public TezTaskAttemptID() {
- taskId = new TezTaskID();
- }
-
- /**
- * Constructs a TaskAttemptID object from given {@link TezTaskID}.
- * @param taskId TaskID that this task belongs to
- * @param id the task attempt number
- */
- public TezTaskAttemptID(TezTaskID taskId, int id) {
- super(id);
- if(taskId == null) {
- throw new IllegalArgumentException("taskId cannot be null");
- }
- this.taskId = taskId;
- }
-
- /** Returns the {@link TezTaskID} object that this task attempt belongs to */
- public TezTaskID getTaskID() {
- return taskId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!super.equals(o))
- return false;
-
- TezTaskAttemptID that = (TezTaskAttemptID)o;
- return this.taskId.equals(that.taskId);
- }
-
- /**
- * Add the unique string to the StringBuilder
- * @param builder the builder to append ot
- * @return the builder that was passed in.
- */
- protected StringBuilder appendTo(StringBuilder builder) {
- return taskId.appendTo(builder).append(SEPARATOR).append(id);
- }
-
- @Override
- public int hashCode() {
- return taskId.hashCode() * 539501 + id;
- }
-
- /**Compare TaskIds by first tipIds, then by task numbers. */
- @Override
- public int compareTo(TezID o) {
- TezTaskAttemptID that = (TezTaskAttemptID)o;
- int tipComp = this.taskId.compareTo(that.taskId);
- if(tipComp == 0) {
- return this.id - that.id;
- }
- else return tipComp;
- }
- @Override
- public String toString() {
- return appendTo(new StringBuilder(ATTEMPT)).toString();
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- taskId.readFields(in);
- super.readFields(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- taskId.write(out);
- super.write(out);
- }
-
- // FIXME TEZ DAG needs to be removed
- public static TezTaskAttemptID read(DataInput in) throws IOException {
- TezTaskAttemptID tId = new TezTaskAttemptID();
- tId.readFields(in);
- return tId;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskID.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskID.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskID.java
deleted file mode 100644
index b6a997d..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskID.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.records;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.text.NumberFormat;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-
-/**
- * TaskID represents the immutable and unique identifier for
- * a Tez Task. Each TaskID encompasses multiple attempts made to
- * execute the Tez Task, each of which are uniquely identified by
- * their TezTaskAttemptID.
- *
- * @see TezTaskAttemptID
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class TezTaskID extends TezID {
- public static final String TASK = "task";
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
- static {
- idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(6);
- }
-
- private TezVertexID vertexId;
-
- public TezTaskID() {
- vertexId = new TezVertexID();
- }
-
- /**
- * Constructs a TaskID object from given {@link MRxApplicationID}.
- * @param jobId JobID that this tip belongs to
- * @param type the {@link TezTaskType} of the task
- * @param id the tip number
- */
- public TezTaskID(TezVertexID vertexId, int id) {
- super(id);
- if(vertexId == null) {
- throw new IllegalArgumentException("vertexId cannot be null");
- }
- this.vertexId = vertexId;
- }
-
- /** Returns the {@link TezVertexID} object that this task belongs to */
- public TezVertexID getVertexID() {
- return vertexId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!super.equals(o))
- return false;
-
- TezTaskID that = (TezTaskID)o;
- return this.vertexId.equals(that.vertexId);
- }
-
- /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
- @Override
- public int compareTo(TezID o) {
- TezTaskID that = (TezTaskID)o;
- int vertexComp = this.vertexId.compareTo(that.vertexId);
- if(vertexComp == 0) {
- return this.id - that.id;
- }
- else return vertexComp;
- }
- @Override
- public String toString() {
- return appendTo(new StringBuilder(TASK)).toString();
- }
-
- /**
- * Add the unique string to the given builder.
- * @param builder the builder to append to
- * @return the builder that was passed in
- */
- protected StringBuilder appendTo(StringBuilder builder) {
- return vertexId.appendTo(builder).
- append(SEPARATOR).
- append(idFormat.format(id));
- }
-
- @Override
- public int hashCode() {
- return vertexId.hashCode() * 535013 + id;
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- vertexId.readFields(in);
- super.readFields(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- vertexId.write(out);
- super.write(out);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskType.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskType.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskType.java
deleted file mode 100644
index 68a5541..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezTaskType.java
+++ /dev/null
@@ -1,26 +0,0 @@
-package org.apache.tez.engine.records;
-///**
-// * Licensed to the Apache Software Foundation (ASF) under one or more
-// * contributor license agreements. See the NOTICE file distributed with this
-// * work for additional information regarding copyright ownership. The ASF
-// * licenses this file to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// * License for the specific language governing permissions and limitations under
-// * the License.
-// */
-//
-//package org.apache.tez.records;
-//
-//
-//public interface TezTaskType {
-//
-// public String toSerializedString();
-//
-//}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezVertexID.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezVertexID.java b/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezVertexID.java
deleted file mode 100644
index 31acd56..0000000
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/records/TezVertexID.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.engine.records;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.text.NumberFormat;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * TezVertexID represents the immutable and unique identifier for
- * a Vertex in a Tez DAG. Each TezVertexID encompasses multiple Tez Tasks.
- *
- * TezVertezID consists of 2 parts. The first part is the {@link TezDAGID},
- * that is the Tez DAG that this vertex belongs to. The second part is
- * the vertex number.
- *
- * @see TezDAGID
- * @see TezTaskID
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class TezVertexID extends TezID {
- public static final String VERTEX = "vertex";
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
- static {
- idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(6);
- }
-
- private TezDAGID dagId;
-
- public TezVertexID() {
- }
-
- /**
- * Constructs a TaskID object from given {@link TezDAGID}.
- * @param applicationId JobID that this tip belongs to
- * @param type the {@link TezTaskType} of the task
- * @param id the tip number
- */
- public TezVertexID(TezDAGID dagId, int id) {
- super(id);
- if(dagId == null) {
- throw new IllegalArgumentException("dagId cannot be null");
- }
- this.dagId = dagId;
- }
-
- /** Returns the {@link TezDAGID} object that this tip belongs to */
- public TezDAGID getDAGId() {
- return dagId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!super.equals(o))
- return false;
-
- TezVertexID that = (TezVertexID)o;
- return this.dagId.equals(that.dagId);
- }
-
- /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
- @Override
- public int compareTo(TezID o) {
- TezVertexID that = (TezVertexID)o;
- return this.dagId.compareTo(that.dagId);
- }
-
- @Override
- public String toString() {
- return appendTo(new StringBuilder(VERTEX)).toString();
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- dagId = new TezDAGID();
- dagId.readFields(in);
- super.readFields(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- dagId.write(out);
- super.write(out);
- }
-
- /**
- * Add the unique string to the given builder.
- * @param builder the builder to append to
- * @return the builder that was passed in
- */
- protected StringBuilder appendTo(StringBuilder builder) {
- return builder.append(SEPARATOR).
- append(dagId.getApplicationId().getClusterTimestamp()).
- append(SEPARATOR).
- append(dagId.getApplicationId().getId()).
- append(SEPARATOR).
- append(dagId.getId()).
- append(SEPARATOR).
- append(idFormat.format(id));
- }
-
- @Override
- public int hashCode() {
- return dagId.hashCode() * 530017 + id;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/pom.xml
----------------------------------------------------------------------
diff --git a/tez-engine/pom.xml b/tez-engine/pom.xml
index ba5b6e6..07bda8c 100644
--- a/tez-engine/pom.xml
+++ b/tez-engine/pom.xml
@@ -34,6 +34,10 @@
<artifactId>tez-common</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-engine-api</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
index 7f825d9..350702a 100644
--- a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
+++ b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
@@ -25,7 +25,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.io.Text;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TezEngineTaskContext extends TezTaskContext {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/common/TezTaskReporter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/common/TezTaskReporter.java b/tez-engine/src/main/java/org/apache/tez/common/TezTaskReporter.java
new file mode 100644
index 0000000..1931e31
--- /dev/null
+++ b/tez-engine/src/main/java/org/apache/tez/common/TezTaskReporter.java
@@ -0,0 +1,116 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.common;
+
+import java.io.IOException;
+
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.util.Progressable;
+import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.engine.api.Master;
+import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
+
+public interface TezTaskReporter extends Progressable, Master {
+
+ public void setStatus(String status);
+
+ public float getProgress();
+
+ public void setProgress(float progress);
+
+ public void progress();
+
+ public TezCounter getCounter(String group, String name);
+
+ public TezCounter getCounter(Enum<?> name);
+
+ public void incrCounter(String group, String counter, long amount);
+
+ public void incrCounter(Enum<?> key, long amount);
+
+ public void reportFatalError(TezTaskAttemptID taskAttemptId,
+ Throwable exception, String logMsg);
+
+ public final TezTaskReporter NULL = new TezTaskReporter() {
+
+ @Override
+ public TezTaskDependencyCompletionEventsUpdate getDependentTasksCompletionEvents(
+ int fromEventIdx, int maxEventsToFetch,
+ TezTaskAttemptID reduce) {
+ return null;
+ }
+
+ @Override
+ public void setStatus(String status) {
+ }
+
+ @Override
+ public void setProgress(float progress) {
+ }
+
+ @Override
+ public void progress() {
+ }
+
+ @Override
+ public void incrCounter(Enum<?> key, long amount) {
+ }
+
+ @Override
+ public void incrCounter(String group, String counter, long amount) {
+ }
+
+ @Override
+ public float getProgress() {
+ return 0.0f;
+ }
+
+ @Override
+ public TezCounter getCounter(Enum<?> name) {
+ return null;
+ }
+
+ @Override
+ public TezCounter getCounter(String group, String name) {
+ return null;
+ }
+
+ @Override
+ public void reportFatalError(TezTaskAttemptID taskAttemptId,
+ Throwable exception, String logMsg) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
+ // TODO TEZAM3
+ return 0;
+ }
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ // TODO TEZAM3
+ return null;
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java b/tez-engine/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
new file mode 100644
index 0000000..e94fd30
--- /dev/null
+++ b/tez-engine/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.tez.common.records.ProceedToCompletionResponse;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.engine.api.Master;
+import org.apache.tez.engine.records.OutputContext;
+
+/** Protocol that task child process uses to contact its parent process. The
+ * parent is a daemon which which polls the central master for a new map or
+ * reduce task and runs it as a child process. All communication between child
+ * and parent is via this protocol. */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+public interface TezTaskUmbilicalProtocol extends Master {
+
+ public static final long versionID = 19L;
+
+ ContainerTask getTask(ContainerContext containerContext) throws IOException;
+
+ boolean statusUpdate(TezTaskAttemptID taskId, TezTaskStatus taskStatus)
+ throws IOException, InterruptedException;
+
+ void reportDiagnosticInfo(TezTaskAttemptID taskid, String trace) throws IOException;
+
+ boolean ping(TezTaskAttemptID taskid) throws IOException;
+
+ void done(TezTaskAttemptID taskid) throws IOException;
+
+ void commitPending(TezTaskAttemptID taskId, TezTaskStatus taskStatus)
+ throws IOException, InterruptedException;
+
+ boolean canCommit(TezTaskAttemptID taskid) throws IOException;
+
+ void shuffleError(TezTaskAttemptID taskId, String message) throws IOException;
+
+ void fsError(TezTaskAttemptID taskId, String message) throws IOException;
+
+ void fatalError(TezTaskAttemptID taskId, String message) throws IOException;
+
+ // TODO TEZAM5 Can commitPending and outputReady be collapsed into a single
+ // call.
+ // IAC outputReady followed by commit is a little confusing - since the output
+ // isn't really in place till a commit is called. Maybe rename to
+ // processingComplete or some such.
+
+ // TODO EVENTUALLY This is not the most useful API. Once there's some kind of
+ // support for the Task handing output over to the Container, this won't rally
+ // be required. i.e. InMemShuffle running as a service in the Container, or
+ // the second task in getTask(). ContainerUmbilical would include getTask and
+ // getServices...
+
+ void outputReady(TezTaskAttemptID taskAttemptId, OutputContext outputContext)
+ throws IOException;
+
+ ProceedToCompletionResponse
+ proceedToCompletion(TezTaskAttemptID taskAttemptId) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/EventFetcher.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/EventFetcher.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/EventFetcher.java
index ac8ea27..9280423 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/EventFetcher.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/EventFetcher.java
@@ -22,9 +22,9 @@ import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.api.Master;
import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
class EventFetcher extends Thread {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java
index 6f6851b..9e905e1 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Fetcher.java
@@ -49,11 +49,11 @@ import org.apache.tez.common.IDUtils;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezTaskReporter;
import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.common.ConfigUtils;
import org.apache.tez.engine.common.security.SecureShuffleUtils;
import org.apache.tez.engine.common.shuffle.impl.MapOutput.Type;
import org.apache.tez.engine.common.sort.impl.IFileInputStream;
-import org.apache.tez.engine.records.TezTaskAttemptID;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryReader.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryReader.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryReader.java
index 78db3d5..7cea558 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryReader.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryReader.java
@@ -26,9 +26,9 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.common.sort.impl.IFile;
import org.apache.tez.engine.common.sort.impl.IFile.Reader;
-import org.apache.tez.engine.records.TezTaskAttemptID;
/**
* <code>IFile.InMemoryReader</code> to read map-outputs present in-memory.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapHost.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapHost.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapHost.java
index 4c12e2a..544f3b5 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapHost.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapHost.java
@@ -20,7 +20,7 @@ package org.apache.tez.engine.common.shuffle.impl;
import java.util.ArrayList;
import java.util.List;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
class MapHost {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapOutput.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapOutput.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapOutput.java
index 41820b3..272709e 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapOutput.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MapOutput.java
@@ -29,8 +29,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BoundedByteArrayOutputStream;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.common.task.local.output.TezTaskOutputFiles;
-import org.apache.tez.engine.records.TezTaskAttemptID;
class MapOutput {
private static final Log LOG = LogFactory.getLog(MapOutput.class);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java
index 9156f28..16ded35 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/MergeManager.java
@@ -45,6 +45,8 @@ import org.apache.tez.common.Constants;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezTaskReporter;
import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.engine.api.Input;
import org.apache.tez.engine.api.Output;
import org.apache.tez.engine.api.Processor;
@@ -57,8 +59,6 @@ import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator;
import org.apache.tez.engine.common.sort.impl.IFile.Writer;
import org.apache.tez.engine.common.sort.impl.TezMerger.Segment;
import org.apache.tez.engine.common.task.local.output.TezTaskOutputFiles;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
@InterfaceAudience.Private
@InterfaceStability.Unstable
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleClientMetrics.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleClientMetrics.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleClientMetrics.java
index 0188f21..34b26c4 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleClientMetrics.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleClientMetrics.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.metrics.MetricsUtil;
import org.apache.hadoop.metrics.Updater;
import org.apache.tez.common.Constants;
import org.apache.tez.common.TezJobConfig;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
class ShuffleClientMetrics implements Updater {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java
index d0ce5f5..de20621 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/ShuffleScheduler.java
@@ -39,8 +39,8 @@ import org.apache.hadoop.util.Progress;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezTaskStatus;
import org.apache.tez.common.counters.TezCounter;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
class ShuffleScheduler {
static ThreadLocal<Long> shuffleStart = new ThreadLocal<Long>() {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
index f6af426..76fc2e9 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
@@ -45,6 +45,7 @@ import org.apache.tez.common.TezEngineTaskContext;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.counters.TaskCounter;
import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.api.Input;
import org.apache.tez.engine.api.Master;
import org.apache.tez.engine.api.Output;
@@ -58,7 +59,6 @@ import org.apache.tez.engine.common.sort.impl.IFile.Writer;
import org.apache.tez.engine.common.task.local.output.TezTaskOutput;
import org.apache.tez.engine.common.task.local.output.TezTaskOutputFiles;
import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
@SuppressWarnings({"unchecked", "rawtypes"})
public abstract class ExternalSorter {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java
index 6ce39da..4da5ffa 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/PipelinedSorter.java
@@ -45,13 +45,13 @@ import org.apache.hadoop.util.IndexedSorter;
import org.apache.hadoop.util.Progress;
import org.apache.tez.common.TezEngineTaskContext;
import org.apache.tez.common.TezJobConfig;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.api.Master;
import org.apache.tez.engine.common.ConfigUtils;
import org.apache.tez.engine.common.sort.SortingOutput;
import org.apache.tez.engine.common.sort.impl.IFile.Writer;
import org.apache.tez.engine.common.sort.impl.TezMerger.Segment;
import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
@SuppressWarnings({"unchecked", "rawtypes"})
public class PipelinedSorter extends ExternalSorter implements SortingOutput {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
index 0c35760..2cf5ab4 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.tez.common.TezEngineTaskContext;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezTaskContext;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.api.Master;
import org.apache.tez.engine.common.ConfigUtils;
import org.apache.tez.engine.common.sort.impl.ExternalSorter;
@@ -53,7 +54,6 @@ import org.apache.tez.engine.common.sort.impl.TezSpillRecord;
import org.apache.tez.engine.common.sort.impl.IFile.Writer;
import org.apache.tez.engine.common.sort.impl.TezMerger.Segment;
import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
@SuppressWarnings({"unchecked", "rawtypes"})
public class DefaultSorter extends ExternalSorter implements IndexedSortable {
[20/50] [abbrv] git commit: TEZ-92. Pass m/r java-opts to DAG plan
Posted by ss...@apache.org.
TEZ-92. Pass m/r java-opts to DAG plan
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/1b0fc357
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/1b0fc357
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/1b0fc357
Branch: refs/heads/master
Commit: 1b0fc35734a07defe05d87b27db5188eb989901a
Parents: 9fb2170
Author: Mike Liddell <ml...@apache.org>
Authored: Tue May 21 13:22:27 2013 -0700
Committer: Mike Liddell <ml...@apache.org>
Committed: Tue May 21 13:23:57 2013 -0700
----------------------------------------------------------------------
.../java/org/apache/tez/mapreduce/YARNRunner.java | 51 ++++++++++++++-
1 files changed, 49 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/1b0fc357/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index fcb2ca1..7582a5e 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -509,7 +509,7 @@ public class YARNRunner implements ClientProtocol {
}
}
- private Vertex configureReduceStage(FileSystem fs, JobID jobId,
+ private Vertex configureIntermediateReduceStage(FileSystem fs, JobID jobId,
Configuration jobConf, String jobSubmitDir, Credentials ts,
Map<String, LocalResource> jobLocalResources, int iReduceIndex)
throws IOException {
@@ -538,6 +538,8 @@ public class YARNRunner implements ClientProtocol {
vertex.setTaskLocationsHint(null);
vertex.setTaskResource(reduceResource);
+ vertex.setJavaOpts(getReduceJavaOpts(conf));
+
return vertex;
}
@@ -551,9 +553,15 @@ public class YARNRunner implements ClientProtocol {
Vertex[] vertices = new Vertex[numIntermediateStages];
for (int i = 0; i < numIntermediateStages; i++) {
- vertices[i] = configureReduceStage(fs, jobId, jobConf, jobSubmitDir, ts,
+ vertices[i] = configureIntermediateReduceStage(fs, jobId, jobConf, jobSubmitDir, ts,
jobLocalResources, i);
dag.addVertex(vertices[i]);
+
+ LOG.info("XXXX Adding intermediate vertex to DAG"
+ + ", vertexName=" + vertices[i].getVertexName()
+ + ", processor=" + vertices[i].getProcessorName()
+ + ", parrellism=" + vertices[i].getParallelism()
+ + ", javaOpts=" + vertices[i].getJavaOpts());
}
return vertices;
}
@@ -604,6 +612,8 @@ public class YARNRunner implements ClientProtocol {
mapVertex.setTaskLocationsHint(inputSplitLocations);
mapVertex.setTaskResource(mapResource);
+ mapVertex.setJavaOpts(getMapJavaOpts(jobConf));
+
LOG.info("XXXX Adding map vertex to DAG"
+ ", vertexName=" + mapVertex.getVertexName()
+ ", processor=" + mapVertex.getProcessorName()
@@ -645,6 +655,8 @@ public class YARNRunner implements ClientProtocol {
reduceVertex.setTaskLocationsHint(null);
reduceVertex.setTaskResource(reduceResource);
+ reduceVertex.setJavaOpts(getReduceJavaOpts(jobConf));
+
LOG.info("XXXX Adding reduce vertex to DAG"
+ ", vertexName=" + reduceVertex.getVertexName()
+ ", processor=" + reduceVertex.getProcessorName()
@@ -1114,4 +1126,39 @@ public class YARNRunner implements ClientProtocol {
envConf + " config settings.");
}
}
+
+ private String getMapJavaOpts(Configuration jobConf) {
+ // follows pattern from YARN MapReduceChildJVM.java
+ String adminOpts = "";
+ adminOpts = jobConf.get(
+ MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,
+ MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
+
+ String userOpts = "";
+ userOpts =
+ jobConf.get(
+ MRJobConfig.MAP_JAVA_OPTS, // same as JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS
+ jobConf.get(
+ JobConf.MAPRED_TASK_JAVA_OPTS,
+ JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
+
+ return adminOpts + " " + userOpts;
+ }
+
+ private String getReduceJavaOpts(Configuration jobConf) {
+ // follows pattern from YARN MapReduceChildJVM.java
+ String adminOpts = "";
+ adminOpts = jobConf.get(
+ MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,
+ MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
+
+ String userOpts = "";
+ userOpts =
+ jobConf.get(
+ MRJobConfig.REDUCE_JAVA_OPTS, // same as JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS
+ jobConf.get(
+ JobConf.MAPRED_TASK_JAVA_OPTS,
+ JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
+ return adminOpts + " " + userOpts;
+ }
}
[29/50] [abbrv] git commit: TEZ-47. Get task max attempts from API or
config (bikas)
Posted by ss...@apache.org.
TEZ-47. Get task max attempts from API or config (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/4beb98f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/4beb98f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/4beb98f3
Branch: refs/heads/master
Commit: 4beb98f3158aa2818395c805500c694b9e11c2b2
Parents: b5ebd49
Author: Bikas Saha <bi...@apache.org>
Authored: Fri May 24 12:06:05 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Fri May 24 12:06:34 2013 -0700
----------------------------------------------------------------------
.../org/apache/tez/dag/api/TezConfiguration.java | 5 +++++
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 4 +---
.../org/apache/tez/dag/app/dag/impl/TaskImpl.java | 3 ++-
3 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4beb98f3/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 3018ea1..1dec08a 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -61,6 +61,11 @@ public class TezConfiguration extends Configuration {
public static final String DAG_MAX_TASK_FAILURES_PER_NODE = TEZ_PREFIX
+ "maxtaskfailures.per.node";
public static final int DAG_MAX_TASK_FAILURES_PER_NODE_DEFAULT = 3;
+
+ public static final String DAG_MAX_TASK_ATTEMPTS =
+ DAG_AM_PREFIX + "max.task.attempts";
+ public static final int DAG_MAX_TASK_ATTEMPTS_DEFAULT = 4;
+
public static final String DAG_NODE_BLACKLISTING_ENABLED = TEZ_PREFIX
+ "node-blacklisting.enabled";
public static final boolean DAG_NODE_BLACKLISTING_ENABLED_DEFAULT = true;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4beb98f3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 92ccffe..5d4242d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -101,14 +101,13 @@ public class TaskAttemptImpl implements TaskAttempt,
EventHandler<TaskAttemptEvent> {
// TODO Ensure MAPREDUCE-4457 is factored in. Also MAPREDUCE-4068.
- // TODO Consider TAL registartion in the TaskAttempt instead of the container.
+ // TODO Consider TAL registration in the TaskAttempt instead of the container.
private static final Log LOG = LogFactory.getLog(TaskAttemptImpl.class);
private static final String LINE_SEPARATOR = System
.getProperty("line.separator");
static final TezCounters EMPTY_COUNTERS = new TezCounters();
- private static final long MEMORY_SPLITS_RESOLUTION = 1024; //TODO Make configurable?
protected final TezConfiguration conf;
protected final int partition;
@@ -144,7 +143,6 @@ public class TaskAttemptImpl implements TaskAttempt,
protected final String javaOpts;
protected final boolean isRescheduled;
- private boolean speculatorContainerRequestSent = false;
protected String processorName;
protected static final FailedTransitionHelper FAILED_HELPER =
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4beb98f3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 401f1b0..fb738ce 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -289,7 +289,8 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
writeLock = readWriteLock.writeLock();
this.attempts = Collections.emptyMap();
// TODO TEZ-47 get from conf or API
- maxAttempts = 4;
+ maxAttempts = this.conf.getInt(TezConfiguration.DAG_MAX_TASK_ATTEMPTS,
+ TezConfiguration.DAG_MAX_TASK_ATTEMPTS_DEFAULT);
taskId = new TezTaskID(vertexId, partition);
this.partition = partition;
this.taskAttemptListener = taskAttemptListener;
[21/50] [abbrv] git commit: TEZ-129. Set LocalDirs correctly for
MapReduce tasks. (sseth)
Posted by ss...@apache.org.
TEZ-129. Set LocalDirs correctly for MapReduce tasks. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/37d7a4cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/37d7a4cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/37d7a4cc
Branch: refs/heads/master
Commit: 37d7a4cc3d65aa754c1ce59444731ec944a20381
Parents: 1b0fc35
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue May 21 16:13:00 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue May 21 16:13:21 2013 -0700
----------------------------------------------------------------------
.../apache/tez/mapreduce/task/MRRuntimeTask.java | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/37d7a4cc/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index 3b51889..e5dfd66 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.tez.common.TezEngineTaskContext;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
@@ -169,6 +168,7 @@ public class MRRuntimeTask extends RuntimeTask {
}
job.set(MRJobConfig.JOB_LOCAL_DIR, job.get(TezJobConfig.JOB_LOCAL_DIR));
+ job.set(MRConfig.LOCAL_DIR, job.get(TezJobConfig.LOCAL_DIRS));
// setup the child's attempt directories
// Do the task-type specific localization
[46/50] [abbrv] TEZ-175. Cleanup logs.
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
index 4c73559..a79358b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java
@@ -55,12 +55,12 @@ public class AMNodeImpl implements AMNode {
private int numSuccessfulTAs = 0;
private boolean blacklistingEnabled;
private boolean ignoreBlacklisting = false;
-
+
@SuppressWarnings("rawtypes")
protected EventHandler eventHandler;
private final List<ContainerId> containers = new LinkedList<ContainerId>();
-
+
//Book-keeping only. In case of Health status change.
private final List<ContainerId> pastContainers = new LinkedList<ContainerId>();
@@ -69,8 +69,8 @@ public class AMNodeImpl implements AMNode {
private final StateMachine<AMNodeState, AMNodeEventType, AMNodeEvent> stateMachine;
private static StateMachineFactory
- <AMNodeImpl, AMNodeState, AMNodeEventType, AMNodeEvent>
- stateMachineFactory =
+ <AMNodeImpl, AMNodeState, AMNodeEventType, AMNodeEvent>
+ stateMachineFactory =
new StateMachineFactory<AMNodeImpl, AMNodeState, AMNodeEventType, AMNodeEvent>(
AMNodeState.ACTIVE)
// Transitions from ACTIVE state.
@@ -97,7 +97,7 @@ public class AMNodeImpl implements AMNode {
.addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.UNHEALTHY, AMNodeEventType.N_TURNED_UNHEALTHY, new NodeTurnedUnhealthyTransition())
.addTransition(AMNodeState.FORCED_ACTIVE, EnumSet.of(AMNodeState.BLACKLISTED, AMNodeState.ACTIVE), AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED, new IgnoreBlacklistingDisabledTransition())
.addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.FORCED_ACTIVE, EnumSet.of(AMNodeEventType.N_TURNED_HEALTHY, AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED), new GenericErrorTransition())
-
+
// Transitions from UNHEALTHY state.
.addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, AMNodeEventType.N_CONTAINER_ALLOCATED, new ContainerAllocatedWhileUnhealthyTransition())
.addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, EnumSet.of(AMNodeEventType.N_TA_SUCCEEDED, AMNodeEventType.N_TA_ENDED))
@@ -107,7 +107,7 @@ public class AMNodeImpl implements AMNode {
.addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, AMNodeEventType.N_TURNED_UNHEALTHY, new GenericErrorTransition())
.installTopology();
-
+
@SuppressWarnings("rawtypes")
public AMNodeImpl(NodeId nodeId, int maxTaskFailuresPerNode,
@@ -153,10 +153,12 @@ public class AMNodeImpl implements AMNode {
@Override
public void handle(AMNodeEvent event) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing AMNodeEvent " + event.getNodeId()
+ + " of type " + event.getType() + " while in state: " + getState()
+ + ". Event: " + event);
+ }
this.writeLock.lock();
- LOG.info("DEBUG: Processing AMNodeEvent " + event.getNodeId()
- + " of type " + event.getType() + " while in state: " + getState()
- + ". Event: " + event);
try {
final AMNodeState oldState = getState();
try {
@@ -174,7 +176,7 @@ public class AMNodeImpl implements AMNode {
writeLock.unlock();
}
}
-
+
protected boolean shouldBlacklistNode() {
return blacklistingEnabled && (numFailedTAs >= maxTaskFailuresPerNode);
}
@@ -193,7 +195,7 @@ public class AMNodeImpl implements AMNode {
//////////////////////////////////////////////////////////////////////////////
// Start of Transition Classes //
//////////////////////////////////////////////////////////////////////////////
-
+
protected static class ContainerAllocatedTransition implements
SingleArcTransition<AMNodeImpl, AMNodeEvent> {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryWriter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryWriter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryWriter.java
index f683605..b3ebb8b 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryWriter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/InMemoryWriter.java
@@ -35,62 +35,64 @@ import org.apache.tez.engine.common.sort.impl.IFile.Writer;
@InterfaceStability.Unstable
public class InMemoryWriter extends Writer {
private static final Log LOG = LogFactory.getLog(InMemoryWriter.class);
-
+
private DataOutputStream out;
-
+
public InMemoryWriter(BoundedByteArrayOutputStream arrayStream) {
super(null);
- this.out =
+ this.out =
new DataOutputStream(new IFileOutputStream(arrayStream));
}
-
+
public void append(Object key, Object value) throws IOException {
throw new UnsupportedOperationException
("InMemoryWriter.append(K key, V value");
}
-
+
public void append(DataInputBuffer key, DataInputBuffer value)
throws IOException {
int keyLength = key.getLength() - key.getPosition();
if (keyLength < 0) {
- throw new IOException("Negative key-length not allowed: " + keyLength +
+ throw new IOException("Negative key-length not allowed: " + keyLength +
" for " + key);
}
-
+
boolean sameKey = (key == IFile.REPEAT_KEY);
-
+
int valueLength = value.getLength() - value.getPosition();
if (valueLength < 0) {
- throw new IOException("Negative value-length not allowed: " +
+ throw new IOException("Negative value-length not allowed: " +
valueLength + " for " + value);
}
-
+
if(sameKey) {
WritableUtils.writeVInt(out, IFile.RLE_MARKER);
WritableUtils.writeVInt(out, valueLength);
out.write(value.getData(), value.getPosition(), valueLength);
} else {
- LOG.info("XXX InMemWriter.append" +
- " key.data=" + key.getData() +
- " key.pos=" + key.getPosition() +
- " key.len=" +key.getLength() +
- " val.data=" + value.getData() +
- " val.pos=" + value.getPosition() +
- " val.len=" + value.getLength());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("InMemWriter.append" +
+ " key.data=" + key.getData() +
+ " key.pos=" + key.getPosition() +
+ " key.len=" +key.getLength() +
+ " val.data=" + value.getData() +
+ " val.pos=" + value.getPosition() +
+ " val.len=" + value.getLength());
+ }
WritableUtils.writeVInt(out, keyLength);
WritableUtils.writeVInt(out, valueLength);
- out.write(key.getData(), key.getPosition(), keyLength);
- out.write(value.getData(), value.getPosition(), valueLength);
+ out.write(key.getData(), key.getPosition(), keyLength);
+ out.write(value.getData(), value.getPosition(), valueLength);
}
-
+
}
public void close() throws IOException {
// Write EOF_MARKER for key/value length
WritableUtils.writeVInt(out, IFile.EOF_MARKER);
WritableUtils.writeVInt(out, IFile.EOF_MARKER);
-
- // Close the stream
+
+ // Close the stream
out.close();
out = null;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
index 76fc2e9..f5976e3 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/ExternalSorter.java
@@ -86,12 +86,12 @@ public abstract class ExternalSorter {
protected SerializationFactory serializationFactory;
protected Serializer keySerializer;
protected Serializer valSerializer;
-
+
protected IndexedSorter sorter;
// Compression for map-outputs
protected CompressionCodec codec;
-
+
// Counters
protected TezCounter mapOutputByteCounter;
protected TezCounter mapOutputRecordCounter;
@@ -101,41 +101,41 @@ public abstract class ExternalSorter {
public void initialize(Configuration conf, Master master)
throws IOException, InterruptedException {
-
+
this.job = conf;
- LOG.info("TEZ_ENGINE_TASK_ATTEMPT_ID: " +
+ LOG.info("TEZ_ENGINE_TASK_ATTEMPT_ID: " +
job.get(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID));
partitions = task.getOutputSpecList().get(0).getNumOutputs();
-// partitions =
+// partitions =
// job.getInt(
-// TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE,
+// TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE,
// TezJobConfig.DEFAULT_TEZ_ENGINE_TASK_OUTDEGREE);
rfs = ((LocalFileSystem)FileSystem.getLocal(job)).getRaw();
-
+
// sorter
sorter = ReflectionUtils.newInstance(job.getClass(
TezJobConfig.TEZ_ENGINE_INTERNAL_SORTER_CLASS, QuickSort.class,
IndexedSorter.class), job);
-
+
comparator = ConfigUtils.getIntermediateOutputKeyComparator(job);
-
+
// k/v serialization
keyClass = ConfigUtils.getIntermediateOutputKeyClass(job);
valClass = ConfigUtils.getIntermediateOutputValueClass(job);
serializationFactory = new SerializationFactory(job);
keySerializer = serializationFactory.getSerializer(keyClass);
valSerializer = serializationFactory.getSerializer(valClass);
-
+
// counters
- mapOutputByteCounter =
+ mapOutputByteCounter =
runningTaskContext.getTaskReporter().getCounter(TaskCounter.MAP_OUTPUT_BYTES);
mapOutputRecordCounter =
runningTaskContext.getTaskReporter().getCounter(TaskCounter.MAP_OUTPUT_RECORDS);
- fileOutputByteCounter =
+ fileOutputByteCounter =
runningTaskContext.getTaskReporter().
getCounter(TaskCounter.MAP_OUTPUT_MATERIALIZED_BYTES);
- spilledRecordsCounter =
+ spilledRecordsCounter =
runningTaskContext.getTaskReporter().getCounter(TaskCounter.SPILLED_RECORDS);
// compression
if (ConfigUtils.shouldCompressIntermediateOutput(job)) {
@@ -146,14 +146,13 @@ public abstract class ExternalSorter {
codec = null;
}
- // Task outputs
+ // Task outputs
mapOutputFile =
(TezTaskOutput) ReflectionUtils.newInstance(
conf.getClass(
- Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
+ Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
TezTaskOutputFiles.class), conf);
-// LOG.info("XXX mapOutputFile: " + mapOutputFile.getClass());
-
+
// sortPhase
sortPhase = runningTaskContext.getProgress().addPhase("sort", 0.333f);
}
@@ -227,13 +226,8 @@ public abstract class ExternalSorter {
if (!src.renameTo(dst)) {
throw new IOException("Unable to rename " + src + " to " + dst);
}
-// LOG.info("XXX sameVolRename src=" + src + ", dst=" + dst);
}
-// public ExternalSorter() {
-// super();
-// }
-
public ExternalSorter(TezEngineTaskContext tezEngineTask) {
this.task = tezEngineTask;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/IFileOutputStream.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/IFileOutputStream.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/IFileOutputStream.java
index 75fcd68..00b8958 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/IFileOutputStream.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/IFileOutputStream.java
@@ -36,11 +36,9 @@ import org.apache.hadoop.util.DataChecksum;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class IFileOutputStream extends FilterOutputStream {
-
- private static final Log LOG = LogFactory.getLog(IFileOutputStream.class);
-
+
/**
- * The output stream to be checksummed.
+ * The output stream to be checksummed.
*/
private final DataChecksum sum;
private byte[] barray;
@@ -62,7 +60,7 @@ public class IFileOutputStream extends FilterOutputStream {
buffer = new byte[4096];
offset = 0;
}
-
+
@Override
public void close() throws IOException {
if (closed) {
@@ -105,8 +103,8 @@ public class IFileOutputStream extends FilterOutputStream {
// FIXME if needed re-enable this in debug mode
if (LOG.isDebugEnabled()) {
LOG.debug("XXX checksum" +
- " b=" + b + " off=" + off +
- " buffer=" + " offset=" + offset +
+ " b=" + b + " off=" + off +
+ " buffer=" + " offset=" + offset +
" len=" + len);
}
*/
@@ -123,7 +121,7 @@ public class IFileOutputStream extends FilterOutputStream {
checksum(b, off, len);
out.write(b,off,len);
}
-
+
@Override
public void write(int b) throws IOException {
barray[0] = (byte) (b & 0xFF);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
index e567492..adbff22 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/dflt/DefaultSorter.java
@@ -57,16 +57,16 @@ import org.apache.tez.engine.records.OutputContext;
@SuppressWarnings({"unchecked", "rawtypes"})
public class DefaultSorter extends ExternalSorter implements IndexedSortable {
-
+
private static final Log LOG = LogFactory.getLog(DefaultSorter.class);
-
+
/**
* The size of each record in the index file for the map-outputs.
*/
public static final int MAP_OUTPUT_INDEX_RECORD_LENGTH = 24;
private final static int APPROX_HEADER_LENGTH = 150;
-
+
// k/v accounting
IntBuffer kvmeta; // metadata overlay on backing store
int kvstart; // marks origin of spill metadata
@@ -118,15 +118,15 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
}
@Override
- public void initialize(Configuration conf, Master master)
+ public void initialize(Configuration conf, Master master)
throws IOException, InterruptedException {
if (task == null) {
LOG.info("Bailing!", new IOException());
return;
}
-
+
super.initialize(conf, master);
-
+
// sanity checks
final float spillper = job.getFloat(
TezJobConfig.TEZ_ENGINE_SORT_SPILL_PERCENT,
@@ -144,7 +144,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
indexCacheMemoryLimit = job.getInt(TezJobConfig.TEZ_ENGINE_INDEX_CACHE_MEMORY_LIMIT_BYTES,
TezJobConfig.DEFAULT_TEZ_ENGINE_INDEX_CACHE_MEMORY_LIMIT_BYTES);
-
+
// buffers and accounting
int maxMemUsage = sortmb << 20;
maxMemUsage -= maxMemUsage % METASIZE;
@@ -169,8 +169,8 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
// k/v serialization
valSerializer.open(bb);
- keySerializer.open(bb);
-
+ keySerializer.open(bb);
+
spillInProgress = false;
minSpillsForCombine = job.getInt(TezJobConfig.TEZ_ENGINE_COMBINE_MIN_SPILLS, 3);
spillThread.setDaemon(true);
@@ -193,7 +193,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
}
@Override
- public void write(Object key, Object value)
+ public void write(Object key, Object value)
throws IOException, InterruptedException {
collect(
key, value, partitioner.getPartition(key, value, partitions));
@@ -673,7 +673,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
spillLock.unlock();
sortAndSpill();
} catch (Throwable t) {
- LOG.warn("ZZZZ: Got an exception in sortAndSpill", t);
+ LOG.warn("Got an exception in sortAndSpill", t);
sortSpillException = t;
} finally {
spillLock.lock();
@@ -727,23 +727,23 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
int getMetaStart() {
return kvend / NMETA;
}
-
+
int getMetaEnd() {
return 1 + // kvend is a valid record
(kvstart >= kvend
? kvstart
: kvmeta.capacity() + kvstart) / NMETA;
}
-
- protected void sortAndSpill()
+
+ protected void sortAndSpill()
throws IOException, InterruptedException {
final int mstart = getMetaStart();
final int mend = getMetaEnd();
sorter.sort(this, mstart, mend, runningTaskContext.getTaskReporter());
- spill(mstart, mend);
+ spill(mstart, mend);
}
-
- protected void spill(int mstart, int mend)
+
+ protected void spill(int mstart, int mend)
throws IOException, InterruptedException {
//approximate the length of the output file to be the length of the
@@ -775,7 +775,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
kvmeta.get(offsetFor(spindex) + PARTITION) == i) {
final int kvoff = offsetFor(spindex);
key.reset(
- kvbuffer,
+ kvbuffer,
kvmeta.get(kvoff + KEYSTART),
(kvmeta.get(kvoff + VALSTART) - kvmeta.get(kvoff + KEYSTART))
);
@@ -795,7 +795,9 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
if (spstart != spindex) {
TezRawKeyValueIterator kvIter =
new MRResultIterator(spstart, spindex);
- LOG.info("DEBUG: Running combine processor");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Running combine processor");
+ }
runCombineProcessor(kvIter, writer);
}
}
@@ -804,10 +806,10 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
writer.close();
// record offsets
- final TezIndexRecord rec =
+ final TezIndexRecord rec =
new TezIndexRecord(
- segmentStart,
- writer.getRawLength(),
+ segmentStart,
+ writer.getRawLength(),
writer.getCompressedLength());
spillRec.putIndex(rec, i);
@@ -870,10 +872,10 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
writer.close();
// record offsets
- TezIndexRecord rec =
+ TezIndexRecord rec =
new TezIndexRecord(
- segmentStart,
- writer.getRawLength(),
+ segmentStart,
+ writer.getRawLength(),
writer.getCompressedLength());
spillRec.putIndex(rec, i);
@@ -913,7 +915,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
: (bufvoid - kvmeta.get(kvoff + VALSTART)) + nextindex;
return vallen;
}
-
+
/**
* Given an offset, populate vbytes with the associated set of
* deserialized value bytes. Should only be called during a spill.
@@ -936,7 +938,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
public InMemValBytes(int bufvoid) {
this.bufvoid = bufvoid;
}
-
+
public void reset(byte[] buffer, int start, int length) {
this.buffer = buffer;
this.start = start;
@@ -957,7 +959,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
InMemValBytes createInMemValBytes() {
return new InMemValBytes(bufvoid);
}
-
+
protected class MRResultIterator implements TezRawKeyValueIterator {
private final DataInputBuffer keybuf = new DataInputBuffer();
private final InMemValBytes vbytes = createInMemValBytes();
@@ -1031,7 +1033,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
if (numSpills == 0) {
//create dummy files
-
+
TezSpillRecord sr = new TezSpillRecord(partitions);
try {
for (int i = 0; i < partitions; i++) {
@@ -1039,11 +1041,11 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
Writer writer =
new Writer(job, finalOut, keyClass, valClass, codec, null);
writer.close();
-
- TezIndexRecord rec =
+
+ TezIndexRecord rec =
new TezIndexRecord(
- segmentStart,
- writer.getRawLength(),
+ segmentStart,
+ writer.getRawLength(),
writer.getCompressedLength());
sr.putIndex(rec, i);
}
@@ -1057,7 +1059,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
else {
sortPhase.addPhases(partitions); // Divide sort phase into sub-phases
TezMerger.considerFinalMergeForProgress();
-
+
final TezSpillRecord spillRec = new TezSpillRecord(partitions);
for (int parts = 0; parts < partitions; parts++) {
//create the segments to be merged
@@ -1074,13 +1076,13 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
if (LOG.isDebugEnabled()) {
LOG.debug("MapId=" + mapId + " Reducer=" + parts +
"Spill =" + i + "(" + indexRecord.getStartOffset() + "," +
- indexRecord.getRawLength() + ", " +
+ indexRecord.getRawLength() + ", " +
indexRecord.getPartLength() + ")");
}
}
- int mergeFactor =
- job.getInt(TezJobConfig.TEZ_ENGINE_IO_SORT_FACTOR,
+ int mergeFactor =
+ job.getInt(TezJobConfig.TEZ_ENGINE_IO_SORT_FACTOR,
TezJobConfig.DEFAULT_TEZ_ENGINE_IO_SORT_FACTOR);
// sort the segments only if there are intermediate merges
boolean sortSegments = segmentList.size() > mergeFactor;
@@ -1089,9 +1091,9 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
keyClass, valClass, codec,
segmentList, mergeFactor,
new Path(mapId.toString()),
- (RawComparator)ConfigUtils.getIntermediateOutputKeyComparator(job),
+ (RawComparator)ConfigUtils.getIntermediateOutputKeyComparator(job),
runningTaskContext.getTaskReporter(), sortSegments,
- null, spilledRecordsCounter,
+ null, spilledRecordsCounter,
sortPhase.phase());
//write merged output to disk
@@ -1109,10 +1111,10 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
sortPhase.startNextPhase();
// record offsets
- final TezIndexRecord rec =
+ final TezIndexRecord rec =
new TezIndexRecord(
- segmentStart,
- writer.getRawLength(),
+ segmentStart,
+ writer.getRawLength(),
writer.getCompressedLength());
spillRec.putIndex(rec, parts);
}
@@ -1123,7 +1125,7 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
}
}
}
-
+
@Override
public OutputContext getOutputContext() {
return null;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
index 5fb6519..b7874f0 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
@@ -55,20 +55,22 @@ public class TezTaskOutputFiles extends TezTaskOutput {
}
// assume configured to $localdir/usercache/$user/appcache/$appId
- private LocalDirAllocator lDirAlloc =
+ private LocalDirAllocator lDirAlloc =
new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
private Path getAttemptOutputDir() {
- LOG.info("DEBUG: getAttemptOutputDir: "
- + Constants.TASK_OUTPUT_DIR + "/"
- + conf.get(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getAttemptOutputDir: "
+ + Constants.TASK_OUTPUT_DIR + "/"
+ + conf.get(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID));
+ }
return new Path(Constants.TASK_OUTPUT_DIR,
conf.get(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID));
}
-
+
/**
* Return the path to local map output file created earlier
- *
+ *
* @return path
* @throws IOException
*/
@@ -80,13 +82,13 @@ public class TezTaskOutputFiles extends TezTaskOutput {
/**
* Create a local map output file name.
- *
+ *
* @param size the size of the file
* @return path
* @throws IOException
*/
public Path getOutputFileForWrite(long size) throws IOException {
- Path attemptOutput =
+ Path attemptOutput =
new Path(getAttemptOutputDir(), Constants.TEZ_ENGINE_TASK_OUTPUT_FILENAME_STRING);
return lDirAlloc.getLocalPathForWrite(attemptOutput.toString(), size, conf);
}
@@ -103,7 +105,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
/**
* Return the path to a local map output index file created earlier
- *
+ *
* @return path
* @throws IOException
*/
@@ -116,7 +118,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
/**
* Create a local map output index file name.
- *
+ *
* @param size the size of the file
* @return path
* @throws IOException
@@ -142,7 +144,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
/**
* Return a local map spill file created earlier.
- *
+ *
* @param spillNumber the number
* @return path
* @throws IOException
@@ -155,7 +157,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
/**
* Create a local map spill file name.
- *
+ *
* @param spillNumber the number
* @param size the size of the file
* @return path
@@ -170,7 +172,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
/**
* Return a local map spill index file created earlier
- *
+ *
* @param spillNumber the number
* @return path
* @throws IOException
@@ -183,7 +185,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
/**
* Create a local map spill index file name.
- *
+ *
* @param spillNumber the number
* @param size the size of the file
* @return path
@@ -198,10 +200,10 @@ public class TezTaskOutputFiles extends TezTaskOutput {
/**
* Return a local reduce input file created earlier
- *
+ *
* @param mapId a map task id
* @return path
- * @throws IOException
+ * @throws IOException
*/
public Path getInputFile(int mapId) throws IOException {
throw new UnsupportedOperationException("Incompatible with LocalRunner");
@@ -209,7 +211,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
/**
* Create a local reduce input file name.
- *
+ *
* @param mapId a map task id
* @param size the size of the file
* @return path
@@ -235,5 +237,5 @@ public class TezTaskOutputFiles extends TezTaskOutput {
public Configuration getConf() {
return conf;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java
index 1d2360b..a954f6e 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/lib/output/LocalOnFileSorterOutput.java
@@ -30,28 +30,28 @@ import org.apache.tez.engine.common.task.local.output.TezTaskOutput;
public class LocalOnFileSorterOutput extends OnFileSortedOutput {
private static final Log LOG = LogFactory.getLog(LocalOnFileSorterOutput.class);
-
+
public LocalOnFileSorterOutput(TezEngineTaskContext task) throws IOException {
super(task);
}
@Override
public void close() throws IOException, InterruptedException {
- LOG.info("XXX close");
-
+ LOG.debug("Closing LocalOnFileSorterOutput");
super.close();
-
TezTaskOutput mapOutputFile = sorter.getMapOutput();
FileSystem localFs = FileSystem.getLocal(mapOutputFile.getConf());
Path src = mapOutputFile.getOutputFile();
- Path dst =
+ Path dst =
mapOutputFile.getInputFileForWrite(
sorter.getTaskAttemptId().getTaskID(),
localFs.getFileStatus(src).getLen());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Renaming src = " + src + ", dst = " + dst);
+ }
localFs.rename(src, dst);
- LOG.info("XXX renaming src = " + src + ", dst = " + dst);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
index 900c2f0..1946a62 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
@@ -62,7 +62,7 @@ public class RuntimeUtils {
public static RuntimeTask createRuntimeTask(
TezEngineTaskContext taskContext) {
- LOG.info("TaskContext"
+ LOG.info("Creating a runtime task from TaskContext"
+ ", Processor: " + taskContext.getProcessorName()
+ ", InputCount=" + taskContext.getInputSpecList().size()
+ ", OutputCount=" + taskContext.getOutputSpecList().size());
@@ -78,38 +78,45 @@ public class RuntimeUtils {
Input[] inputs;
Output[] outputs;
if (taskContext.getInputSpecList().isEmpty()) {
- LOG.info("Initializing task with 0 inputs");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initializing task with 0 inputs");
+ }
inputs = new Input[0];
} else {
int iSpecCount = taskContext.getInputSpecList().size();
inputs = new Input[iSpecCount];
for (int i = 0; i < iSpecCount; ++i) {
InputSpec inSpec = taskContext.getInputSpecList().get(i);
- LOG.info("XXXX Using Input"
- + ", index=" + i
- + ", inputClass=" + inSpec.getInputClassName());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using Input"
+ + ", index=" + i
+ + ", inputClass=" + inSpec.getInputClassName());
+ }
Class<?> inputClazz = Class.forName(inSpec.getInputClassName());
Input input = (Input) getNewInstance(inputClazz, taskContext);
inputs[i] = input;
}
}
if (taskContext.getOutputSpecList().isEmpty()) {
- LOG.info("Initializing task with 0 outputs");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initializing task with 0 outputs");
+ }
outputs = new Output[0];
} else {
int oSpecCount = taskContext.getOutputSpecList().size();
outputs = new Output[oSpecCount];
for (int i = 0; i < oSpecCount; ++i) {
OutputSpec outSpec = taskContext.getOutputSpecList().get(i);
- LOG.info("XXXX Using Output"
- + ", index=" + i
- + ", output=" + outSpec.getOutputClassName());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using Output"
+ + ", index=" + i
+ + ", output=" + outSpec.getOutputClassName());
+ }
Class<?> outputClazz = Class.forName(outSpec.getOutputClassName());
Output output = (Output) getNewInstance(outputClazz, taskContext);
outputs[i] = output;
}
}
- // t = new RuntimeTask(taskContext, processor, inputs, outputs);
t = createRuntime(taskContext, processor, inputs, outputs);
} catch (ClassNotFoundException e) {
throw new YarnException("Unable to initialize RuntimeTask, context="
@@ -117,7 +124,7 @@ public class RuntimeUtils {
}
return t;
}
-
+
private static RuntimeTask createRuntime(TezEngineTaskContext taskContext,
Processor processor, Input[] inputs, Output[] outputs) {
try {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 5dade2b..e930044 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -106,7 +106,7 @@ public class LocalJobRunner implements ClientProtocol {
private AtomicInteger map_tasks = new AtomicInteger(0);
private int reduce_tasks = 0;
final Random rand = new Random();
-
+
private LocalJobRunnerMetrics myMetrics = null;
private static final String jobDir = "localRunner/";
@@ -129,7 +129,7 @@ public class LocalJobRunner implements ClientProtocol {
// This is analogous to JobTracker's system directory.
private Path systemJobDir;
private Path systemJobFile;
-
+
// The job directory for the task. Analagous to a task's job directory.
private Path localJobDir;
private Path localJobFile;
@@ -149,13 +149,13 @@ public class LocalJobRunner implements ClientProtocol {
private JobProfile profile;
private FileSystem localFs;
boolean killed = false;
-
+
private LocalDistributedCacheManager localDistributedCacheManager;
public long getProtocolVersion(String protocol, long clientVersion) {
return TaskUmbilicalProtocol.versionID;
}
-
+
@Override
public ProtocolSignature getProtocolSignature(String protocol,
long clientVersion, int clientMethodsHash) throws IOException {
@@ -176,7 +176,7 @@ public class LocalJobRunner implements ClientProtocol {
// this will trigger localFile to be re-written again.
localDistributedCacheManager = new LocalDistributedCacheManager();
localDistributedCacheManager.setup(conf);
-
+
// Write out configuration file. Instead of copying it from
// systemJobFile, we re-write it, since setup(), above, may have
// updated it.
@@ -193,11 +193,11 @@ public class LocalJobRunner implements ClientProtocol {
setContextClassLoader(localDistributedCacheManager.makeClassLoader(
getContextClassLoader()));
}
-
- profile = new JobProfile(job.getUser(), id, systemJobFile.toString(),
+
+ profile = new JobProfile(job.getUser(), id, systemJobFile.toString(),
"http://localhost:8080/", job.getJobName());
- status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING,
- profile.getUser(), profile.getJobName(), profile.getJobFile(),
+ status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING,
+ profile.getUser(), profile.getJobName(), profile.getJobFile(),
profile.getURL().toString());
jobs.put(id, this);
@@ -235,7 +235,7 @@ public class LocalJobRunner implements ClientProtocol {
TaskAttemptID mapId = new TaskAttemptID(new TaskID(
jobId, TaskType.MAP, taskId), 0);
LOG.info("Starting task: " + mapId);
- final String user =
+ final String user =
UserGroupInformation.getCurrentUser().getShortUserName();
setupChildMapredLocalDirs(mapId, user, localConf);
localConf.setUser(user);
@@ -244,7 +244,7 @@ public class LocalJobRunner implements ClientProtocol {
IDConverter.fromMRTaskAttemptId(mapId);
mapIds.add(mapId);
// FIXME invalid task context
- TezEngineTaskContext taskContext =
+ TezEngineTaskContext taskContext =
new TezEngineTaskContext(
tezMapId, user, localConf.getJobName(), "TODO_vertexName",
MapProcessor.class.getName(),
@@ -346,7 +346,7 @@ public class LocalJobRunner implements ClientProtocol {
return executor;
}
- private org.apache.hadoop.mapreduce.OutputCommitter
+ private org.apache.hadoop.mapreduce.OutputCommitter
createOutputCommitter(boolean newApiCommitter, JobID jobId, Configuration conf) throws Exception {
org.apache.hadoop.mapreduce.OutputCommitter committer = null;
@@ -358,7 +358,7 @@ public class LocalJobRunner implements ClientProtocol {
new org.apache.hadoop.mapreduce.TaskID(jobId, TaskType.MAP, 0);
org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID =
new org.apache.hadoop.mapreduce.TaskAttemptID(taskId, 0);
- org.apache.hadoop.mapreduce.TaskAttemptContext taskContext =
+ org.apache.hadoop.mapreduce.TaskAttemptContext taskContext =
new TaskAttemptContextImpl(conf, taskAttemptID);
@SuppressWarnings("rawtypes")
OutputFormat outputFormat =
@@ -377,7 +377,7 @@ public class LocalJobRunner implements ClientProtocol {
public void run() {
JobID jobId = profile.getJobID();
JobContext jContext = new JobContextImpl(job, jobId);
-
+
org.apache.hadoop.mapreduce.OutputCommitter outputCommitter = null;
try {
outputCommitter = createOutputCommitter(conf.getUseNewMapper(), jobId, conf);
@@ -385,9 +385,9 @@ public class LocalJobRunner implements ClientProtocol {
LOG.info("Failed to createOutputCommitter", e);
return;
}
-
+
try {
- TaskSplitMetaInfo[] taskSplitMetaInfos =
+ TaskSplitMetaInfo[] taskSplitMetaInfos =
SplitMetaInfoReader.readSplitMetaInfo(jobId, localFs, conf, systemJobDir);
int numReduceTasks = job.getNumReduceTasks();
@@ -440,7 +440,7 @@ public class LocalJobRunner implements ClientProtocol {
LOG.info("Starting task: " + reduceId);
try {
if (numReduceTasks > 0) {
- String user =
+ String user =
UserGroupInformation.getCurrentUser().getShortUserName();
JobConf localConf = new JobConf(job);
localConf.setUser(user);
@@ -457,14 +457,16 @@ public class LocalJobRunner implements ClientProtocol {
Collections.singletonList(new OutputSpec("TODO_targetVertex",
0, SimpleOutput.class.getName())));
- // move map output to reduce input
+ // move map output to reduce input
for (int i = 0; i < mapIds.size(); i++) {
if (!this.isInterrupted()) {
TaskAttemptID mapId = mapIds.get(i);
- LOG.info("XXX mapId: " + i +
- " LOCAL_DIR = " +
- mapOutputFiles.get(mapId).getConf().get(
- TezJobConfig.LOCAL_DIRS));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("XXX mapId: " + i +
+ " LOCAL_DIR = " +
+ mapOutputFiles.get(mapId).getConf().get(
+ TezJobConfig.LOCAL_DIRS));
+ }
Path mapOut = mapOutputFiles.get(mapId).getOutputFile();
TezTaskOutput localOutputFile = new TezLocalTaskOutputFiles();
localOutputFile.setConf(localConf);
@@ -514,7 +516,7 @@ public class LocalJobRunner implements ClientProtocol {
} catch (Throwable t) {
try {
- outputCommitter.abortJob(jContext,
+ outputCommitter.abortJob(jContext,
org.apache.hadoop.mapreduce.JobStatus.State.FAILED);
} catch (IOException ioe) {
LOG.info("Error cleaning up job:" + id);
@@ -547,7 +549,7 @@ public class LocalJobRunner implements ClientProtocol {
throws IOException {
return null;
}
-
+
@Override
public synchronized boolean statusUpdate(TezTaskAttemptID taskId,
TezTaskStatus taskStatus) throws IOException, InterruptedException {
@@ -596,7 +598,7 @@ public class LocalJobRunner implements ClientProtocol {
*/
@Override
public void commitPending(TezTaskAttemptID taskid,
- TezTaskStatus taskStatus)
+ TezTaskStatus taskStatus)
throws IOException, InterruptedException {
statusUpdate(taskid, taskStatus);
}
@@ -605,17 +607,17 @@ public class LocalJobRunner implements ClientProtocol {
public void reportDiagnosticInfo(TezTaskAttemptID taskid, String trace) {
// Ignore for now
}
-
+
@Override
public boolean ping(TezTaskAttemptID taskid) throws IOException {
return true;
}
-
+
@Override
public boolean canCommit(TezTaskAttemptID taskid) throws IOException {
return true;
}
-
+
@Override
public void done(TezTaskAttemptID taskId) throws IOException {
int taskIndex = mapIds.indexOf(taskId);
@@ -627,25 +629,25 @@ public class LocalJobRunner implements ClientProtocol {
}
@Override
- public synchronized void fsError(TezTaskAttemptID taskId, String message)
+ public synchronized void fsError(TezTaskAttemptID taskId, String message)
throws IOException {
LOG.fatal("FSError: "+ message + "from task: " + taskId);
}
@Override
- public void shuffleError(TezTaskAttemptID taskId, String message)
+ public void shuffleError(TezTaskAttemptID taskId, String message)
throws IOException {
LOG.fatal("shuffleError: "+ message + "from task: " + taskId);
}
-
+
@Override
- public synchronized void fatalError(TezTaskAttemptID taskId, String msg)
+ public synchronized void fatalError(TezTaskAttemptID taskId, String msg)
throws IOException {
LOG.fatal("Fatal: "+ msg + "from task: " + taskId);
}
-
+
@Override
- public TezTaskDependencyCompletionEventsUpdate
+ public TezTaskDependencyCompletionEventsUpdate
getDependentTasksCompletionEvents(
int fromEventIdx, int maxEventsToFetch,
TezTaskAttemptID reduce) {
@@ -704,7 +706,7 @@ public class LocalJobRunner implements ClientProtocol {
throw new UnsupportedOperationException("Changing job priority " +
"in LocalJobRunner is not supported.");
}
-
+
/** Throws {@link UnsupportedOperationException} */
public boolean killTask(org.apache.hadoop.mapreduce.TaskAttemptID taskId,
boolean shouldFail) throws IOException {
@@ -722,10 +724,10 @@ public class LocalJobRunner implements ClientProtocol {
Job job = jobs.get(JobID.downgrade(id));
if(job != null)
return job.status;
- else
+ else
return null;
}
-
+
public org.apache.hadoop.mapreduce.Counters getJobCounters(
org.apache.hadoop.mapreduce.JobID id) {
Job job = jobs.get(JobID.downgrade(id));
@@ -737,7 +739,7 @@ public class LocalJobRunner implements ClientProtocol {
public String getFilesystemName() throws IOException {
return fs.getUri().toString();
}
-
+
public ClusterMetrics getClusterMetrics() {
int numMapTasks = map_tasks.get();
return new ClusterMetrics(numMapTasks, reduce_tasks, numMapTasks,
@@ -748,25 +750,25 @@ public class LocalJobRunner implements ClientProtocol {
return JobTrackerStatus.RUNNING;
}
- public long getTaskTrackerExpiryInterval()
+ public long getTaskTrackerExpiryInterval()
throws IOException, InterruptedException {
return 0;
}
- /**
- * Get all active trackers in cluster.
+ /**
+ * Get all active trackers in cluster.
* @return array of TaskTrackerInfo
*/
- public TaskTrackerInfo[] getActiveTrackers()
+ public TaskTrackerInfo[] getActiveTrackers()
throws IOException, InterruptedException {
return null;
}
- /**
- * Get all blacklisted trackers in cluster.
+ /**
+ * Get all blacklisted trackers in cluster.
* @return array of TaskTrackerInfo
*/
- public TaskTrackerInfo[] getBlacklistedTrackers()
+ public TaskTrackerInfo[] getBlacklistedTrackers()
throws IOException, InterruptedException {
return null;
}
@@ -776,10 +778,10 @@ public class LocalJobRunner implements ClientProtocol {
, int fromEventId, int maxEvents) throws IOException {
return TaskCompletionEvent.EMPTY_ARRAY;
}
-
+
public org.apache.hadoop.mapreduce.JobStatus[] getAllJobs() {return null;}
-
+
/**
* Returns the diagnostic information for a particular task in the given job.
* To be implemented
@@ -794,7 +796,7 @@ public class LocalJobRunner implements ClientProtocol {
*/
public String getSystemDir() {
Path sysDir = new Path(
- conf.get(JTConfig.JT_SYSTEM_DIR, "/tmp/hadoop/mapred/system"));
+ conf.get(JTConfig.JT_SYSTEM_DIR, "/tmp/hadoop/mapred/system"));
return fs.makeQualified(sysDir).toString();
}
@@ -809,7 +811,7 @@ public class LocalJobRunner implements ClientProtocol {
* @see org.apache.hadoop.mapreduce.protocol.ClientProtocol#getStagingAreaDir()
*/
public String getStagingAreaDir() throws IOException {
- Path stagingRootDir = new Path(conf.get(JTConfig.JT_STAGING_AREA_ROOT,
+ Path stagingRootDir = new Path(conf.get(JTConfig.JT_STAGING_AREA_ROOT,
"/tmp/hadoop/mapred/staging"));
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
String user;
@@ -820,7 +822,7 @@ public class LocalJobRunner implements ClientProtocol {
}
return fs.makeQualified(new Path(stagingRootDir, user+"/.staging")).toString();
}
-
+
public String getJobHistoryDir() {
return null;
}
@@ -847,7 +849,7 @@ public class LocalJobRunner implements ClientProtocol {
}
@Override
- public org.apache.hadoop.mapreduce.QueueAclsInfo[]
+ public org.apache.hadoop.mapreduce.QueueAclsInfo[]
getQueueAclsForCurrentUser() throws IOException{
return null;
}
@@ -879,7 +881,7 @@ public class LocalJobRunner implements ClientProtocol {
}
@Override
- public Token<DelegationTokenIdentifier>
+ public Token<DelegationTokenIdentifier>
getDelegationToken(Text renewer) throws IOException, InterruptedException {
return null;
}
@@ -896,10 +898,10 @@ public class LocalJobRunner implements ClientProtocol {
throws IOException, InterruptedException {
throw new UnsupportedOperationException("Not supported");
}
-
+
static void setupChildMapredLocalDirs(
TaskAttemptID taskAttemptID, String user, JobConf conf) {
- String[] localDirs =
+ String[] localDirs =
conf.getTrimmedStrings(
TezJobConfig.LOCAL_DIRS, TezJobConfig.DEFAULT_LOCAL_DIRS);
String jobId = taskAttemptID.getJobID().toString();
@@ -912,17 +914,17 @@ public class LocalJobRunner implements ClientProtocol {
childMapredLocalDir.append("," + localDirs[i] + Path.SEPARATOR
+ getLocalTaskDir(user, jobId, taskId, isCleanup));
}
- LOG.info(TezJobConfig.LOCAL_DIRS + " for child : " + taskAttemptID +
+ LOG.info(TezJobConfig.LOCAL_DIRS + " for child : " + taskAttemptID +
" is " + childMapredLocalDir);
conf.set(TezJobConfig.LOCAL_DIRS, childMapredLocalDir.toString());
- conf.setClass(Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
+ conf.setClass(Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
TezLocalTaskOutputFiles.class, TezTaskOutput.class);
}
-
+
static final String TASK_CLEANUP_SUFFIX = ".cleanup";
static final String SUBDIR = jobDir;
static final String JOBCACHE = "jobcache";
-
+
static String getLocalTaskDir(String user, String jobid, String taskid,
boolean isCleanupAttempt) {
String taskDir = SUBDIR + Path.SEPARATOR + user + Path.SEPARATOR + JOBCACHE
@@ -932,6 +934,6 @@ public class LocalJobRunner implements ClientProtocol {
}
return taskDir;
}
-
-
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
index 72246c0..0fcfe65 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java
@@ -61,12 +61,14 @@ public class SplitMetaInfoReaderTez {
conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR),
MRJobConfig.JOB_SPLIT_METAINFO);
String jobSplitFile = MRJobConfig.JOB_SPLIT;
-
+
File file = new File(metaSplitFile.toUri().getPath()).getAbsoluteFile();
- LOG.info("DEBUG: Setting up JobSplitIndex with JobSplitFile at: "
- + file.getAbsolutePath() + ", defaultFS from conf: "
- + FileSystem.getDefaultUri(conf));
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting up JobSplitIndex with JobSplitFile at: "
+ + file.getAbsolutePath() + ", defaultFS from conf: "
+ + FileSystem.getDefaultUri(conf));
+ }
+
FileStatus fStatus = fs.getFileStatus(metaSplitFile);
if (maxMetaInfoSize > 0 && fStatus.getLen() > maxMetaInfoSize) {
throw new IOException("Split metadata size exceeded " + maxMetaInfoSize
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
index b8a99ee..a3880f4 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
@@ -83,8 +83,9 @@ public class MRCombiner implements Processor {
@Override
public void process(Input[] in, Output[] out) throws IOException,
InterruptedException {
- LOG.info("DEBUG: Running MRCombiner"
- + ", usingNewAPI=" + useNewApi);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Running MRCombiner, usingNewAPI=" + useNewApi);
+ }
CombineInput input = (CombineInput)in[0];
CombineOutput output = (CombineOutput)out[0];
@@ -334,8 +335,6 @@ public class MRCombiner implements Processor {
comparator,
keyClass,
valueClass);
- LOG.info("DEBUG: Using combineKeyClass: "
- + keyClass + ", combineValueClass: " + valueClass);
org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE>.Context
reducerContext = new
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
index 9f1ae88..54648f5 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java
@@ -31,7 +31,7 @@ import org.apache.tez.mapreduce.hadoop.DeprecatedKeys.MultiStageKeys;
public class MultiStageMRConfToTezTranslator {
private static final Log LOG = LogFactory.getLog(MultiStageMRConfToTezTranslator.class);
-
+
private enum DeprecationReason {
DEPRECATED_DIRECT_TRANSLATION, DEPRECATED_MULTI_STAGE
}
@@ -55,14 +55,14 @@ public class MultiStageMRConfToTezTranslator {
int numEdges = totalStages - 1;
Configuration[] allConfs = extractStageConfs(newConf, numEdges);
-
+
for (int i = 0; i < allConfs.length; i++) {
setStageKeysFromBaseConf(allConfs[i], srcConf, Integer.toString(i));
processDirectConversion(allConfs[i]);
}
for (int i = 0; i < allConfs.length - 1; i++) {
processMultiStageDepreaction(allConfs[i], allConfs[i + 1]);
-
+
}
// Unset unnecessary keys in the last stage. Will end up being called for
// single stage as well which should be harmless.
@@ -163,29 +163,35 @@ public class MultiStageMRConfToTezTranslator {
JobConf jobConf = new JobConf(baseConf);
// Don't clobber explicit tez config.
if (conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_INPUT_KEY_CLASS) == null
- && conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_OUTPUT_KEY_CLASS) == null) {
+ && conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_OUTPUT_KEY_CLASS)
+ == null) {
// If this is set, but the comparator is not set, and their types differ -
// the job will break.
if (conf.get(MRJobConfig.MAP_OUTPUT_KEY_CLASS) == null) {
// Pull tis in from the baseConf
conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, jobConf
.getMapOutputKeyClass().getName());
- LOG.info("XXX: Setting " + MRJobConfig.MAP_OUTPUT_KEY_CLASS
- + " for stage: " + stage
- + " based on job level configuration. Value: "
- + conf.get(MRJobConfig.MAP_OUTPUT_KEY_CLASS));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting " + MRJobConfig.MAP_OUTPUT_KEY_CLASS
+ + " for stage: " + stage
+ + " based on job level configuration. Value: "
+ + conf.get(MRJobConfig.MAP_OUTPUT_KEY_CLASS));
+ }
}
}
-
+
if (conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_INPUT_VALUE_CLASS) == null
- && conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_OUTPUT_VALUE_CLASS) == null) {
+ && conf.get(TezJobConfig.TEZ_ENGINE_INTERMEDIATE_OUTPUT_VALUE_CLASS)
+ == null) {
if (conf.get(MRJobConfig.MAP_OUTPUT_VALUE_CLASS) == null) {
conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, jobConf
.getMapOutputValueClass().getName());
- LOG.info("XXX: Setting " + MRJobConfig.MAP_OUTPUT_VALUE_CLASS
- + " for stage: " + stage
- + " based on job level configuration. Value: "
- + conf.get(MRJobConfig.MAP_OUTPUT_VALUE_CLASS));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting " + MRJobConfig.MAP_OUTPUT_VALUE_CLASS
+ + " for stage: " + stage
+ + " based on job level configuration. Value: "
+ + conf.get(MRJobConfig.MAP_OUTPUT_VALUE_CLASS));
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
index 276294c..5683fa1 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
@@ -76,16 +76,16 @@ import org.apache.tez.mapreduce.hadoop.mapred.TaskAttemptContextImpl;
import org.apache.tez.mapreduce.hadoop.mapreduce.JobContextImpl;
import org.apache.tez.mapreduce.partition.MRPartitioner;
-public abstract class MRTask
+public abstract class MRTask
extends RunningTaskContext {
static final Log LOG = LogFactory.getLog(MRTask.class);
-
+
protected JobConf jobConf;
protected JobContext jobContext;
protected TaskAttemptContext taskAttemptContext;
protected OutputCommitter committer;
-
+
// Current counters
transient TezCounters counters = new TezCounters();
protected GcTimeUpdater gcUpdater;
@@ -93,10 +93,10 @@ extends RunningTaskContext {
private long initCpuCumulativeTime = 0;
protected TezEngineTaskContext tezEngineTaskContext;
protected TezTaskAttemptID taskAttemptId;
-
+
/* flag to track whether task is done */
AtomicBoolean taskDone = new AtomicBoolean(false);
-
+
/** Construct output file names so that, when an output directory listing is
* sorted lexicographically, positions correspond to output partitions.*/
private static final NumberFormat NUMBER_FORMAT = NumberFormat.getInstance();
@@ -104,16 +104,16 @@ extends RunningTaskContext {
NUMBER_FORMAT.setMinimumIntegerDigits(5);
NUMBER_FORMAT.setGroupingUsed(false);
}
-
+
private final static int MAX_RETRIES = 10;
/** The number of milliseconds between progress reports. */
public static final int PROGRESS_INTERVAL = 3000;
private MRTaskReporter mrReporter;
-
+
protected TaskSplitIndex splitMetaInfo = new TaskSplitIndex();
-
+
/**
* A Map where Key-> URIScheme and value->FileSystemStatisticUpdater
*/
@@ -129,7 +129,7 @@ extends RunningTaskContext {
new MRTaskStatus(
taskAttemptId,
counters,
- (taskAttemptId.getTaskID().getVertexID().getId() == 0 ?
+ (taskAttemptId.getTaskID().getVertexID().getId() == 0 ?
Phase.MAP : Phase.SHUFFLE)
);
gcUpdater = new GcTimeUpdater(counters);
@@ -143,15 +143,15 @@ extends RunningTaskContext {
} else {
this.jobConf = new JobConf(conf);
}
- reporter =
+ reporter =
new TezTaskReporterImpl(this, (TezTaskUmbilicalProtocol)master);
((TezTaskReporterImpl)reporter).startCommunicationThread();
-
- jobConf.set(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID,
+
+ jobConf.set(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID,
taskAttemptId.toString());
initResourceCalculatorPlugin();
-
+
LOG.info("MRTask.inited: taskAttemptId = " + taskAttemptId.toString());
}
@@ -171,7 +171,7 @@ extends RunningTaskContext {
public TezTaskUmbilicalProtocol getUmbilical() {
return ((TezTaskReporterImpl)reporter).getUmbilical();
}
-
+
public void initTask(JobConf job, TezDAGID dagId,
MRTaskReporter mrReporter,
boolean useNewApi) throws IOException,
@@ -203,18 +203,18 @@ extends RunningTaskContext {
} else {
setCommitter(job.getOutputCommitter());
}
-
+
Path outputPath = FileOutputFormat.getOutputPath(job);
if (outputPath != null) {
if ((getCommitter() instanceof FileOutputCommitter)) {
- FileOutputFormat.setWorkOutputPath(job,
+ FileOutputFormat.setWorkOutputPath(job,
((FileOutputCommitter)getCommitter()).getTaskAttemptPath(taskAttemptContext));
} else {
FileOutputFormat.setWorkOutputPath(job, outputPath);
}
}
getCommitter().setupTask(taskAttemptContext);
-
+
partitioner = new MRPartitioner(this);
((MRPartitioner)partitioner).initialize(job, getTaskReporter());
@@ -244,7 +244,7 @@ extends RunningTaskContext {
public void setState(State state) {
// TODO Auto-generated method stub
-
+
}
public State getState() {
@@ -261,22 +261,22 @@ extends RunningTaskContext {
}
public TezCounters getCounters() { return counters; }
-
+
/**
- * Return current phase of the task.
+ * Return current phase of the task.
* needs to be synchronized as communication thread sends the phase every second
* @return the curent phase of the task
*/
public synchronized TezTaskStatus.Phase getPhase(){
- return status.getPhase();
+ return status.getPhase();
}
-
+
/**
- * Set current phase of the task.
- * @param phase task phase
+ * Set current phase of the task.
+ * @param phase task phase
*/
protected synchronized void setPhase(TezTaskStatus.Phase phase){
- status.setPhase(phase);
+ status.setPhase(phase);
}
public void setConf(JobConf jobConf) {
@@ -290,9 +290,9 @@ extends RunningTaskContext {
/**
* Gets a handle to the Statistics instance based on the scheme associated
* with path.
- *
+ *
* @param path the path.
- * @param conf the configuration to extract the scheme from if not part of
+ * @param conf the configuration to extract the scheme from if not part of
* the path.
* @return a Statistics instance, or null if none is found for the scheme.
*/
@@ -313,7 +313,7 @@ extends RunningTaskContext {
public synchronized String getOutputName() {
return "part-" + NUMBER_FORMAT.format(taskAttemptId.getTaskID().getId());
}
-
+
public void waitBeforeCompletion(MRTaskReporter reporter) throws IOException,
InterruptedException {
TezTaskUmbilicalProtocol umbilical = getUmbilical();
@@ -400,7 +400,7 @@ extends RunningTaskContext {
} catch (InterruptedException ie) {
// ignore
} catch (IOException ie) {
- LOG.warn("Failure sending commit pending: " +
+ LOG.warn("Failure sending commit pending: " +
StringUtils.stringifyException(ie));
if (--retries == 0) {
System.exit(67);
@@ -421,7 +421,7 @@ extends RunningTaskContext {
sendDone(umbilical);
}
-
+
private boolean isCommitRequired() throws IOException {
return committer.needsTaskCommit(taskAttemptContext);
}
@@ -444,7 +444,7 @@ extends RunningTaskContext {
} catch (InterruptedException ie) {
Thread.currentThread().interrupt(); // interrupt ourself
} catch (IOException ie) {
- LOG.warn("Failure sending status update: " +
+ LOG.warn("Failure sending status update: " +
StringUtils.stringifyException(ie));
if (--retries == 0) {
throw ie;
@@ -454,9 +454,9 @@ extends RunningTaskContext {
}
/**
- * Sends last status update before sending umbilical.done();
+ * Sends last status update before sending umbilical.done();
*/
- private void sendLastUpdate()
+ private void sendLastUpdate()
throws IOException, InterruptedException {
status.setOutputSize(-1l);
// send a final status report
@@ -485,7 +485,7 @@ extends RunningTaskContext {
}
break;
} catch (IOException ie) {
- LOG.warn("Failure asking whether task can commit: " +
+ LOG.warn("Failure asking whether task can commit: " +
StringUtils.stringifyException(ie));
if (--retries == 0) {
//if it couldn't query successfully then delete the output
@@ -495,13 +495,13 @@ extends RunningTaskContext {
}
}
- // task can Commit now
+ // task can Commit now
try {
LOG.info("Task " + taskAttemptId + " is allowed to commit now");
committer.commitTask(taskAttemptContext);
return;
} catch (IOException iee) {
- LOG.warn("Failure committing: " +
+ LOG.warn("Failure committing: " +
StringUtils.stringifyException(iee));
//if it couldn't commit a successfully then delete the output
discardOutput(taskAttemptContext);
@@ -509,12 +509,12 @@ extends RunningTaskContext {
}
}
- private
+ private
void discardOutput(TaskAttemptContext taskContext) {
try {
committer.abortTask(taskContext);
} catch (IOException ioe) {
- LOG.warn("Failure cleaning up: " +
+ LOG.warn("Failure cleaning up: " +
StringUtils.stringifyException(ioe));
}
}
@@ -528,7 +528,7 @@ extends RunningTaskContext {
LOG.info("Task '" + taskAttemptId + "' done.");
return;
} catch (IOException ie) {
- LOG.warn("Failure signalling completion: " +
+ LOG.warn("Failure signalling completion: " +
StringUtils.stringifyException(ie));
if (--retries == 0) {
throw ie;
@@ -540,7 +540,7 @@ extends RunningTaskContext {
public void updateCounters() {
// TODO Auto-generated method stub
// TODO TEZAM Implement.
- Map<String, List<FileSystem.Statistics>> map = new
+ Map<String, List<FileSystem.Statistics>> map = new
HashMap<String, List<FileSystem.Statistics>>();
for(Statistics stat: FileSystem.getAllStatistics()) {
String uriScheme = stat.getScheme();
@@ -563,7 +563,7 @@ extends RunningTaskContext {
}
updater.updateCounters();
}
-
+
gcUpdater.incrementGcCounter();
updateResourceCounters();
}
@@ -599,7 +599,7 @@ extends RunningTaskContext {
counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem);
counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem);
}
-
+
public static String normalizeStatus(String status, Configuration conf) {
// Check to see if the status string is too long
@@ -614,75 +614,78 @@ extends RunningTaskContext {
}
return status;
}
-
- protected static <INKEY,INVALUE,OUTKEY,OUTVALUE>
+
+ protected static <INKEY,INVALUE,OUTKEY,OUTVALUE>
org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE>.Context
createReduceContext(org.apache.hadoop.mapreduce.Reducer
<INKEY,INVALUE,OUTKEY,OUTVALUE> reducer,
Configuration job,
- TezTaskAttemptID taskId,
+ TezTaskAttemptID taskId,
final TezRawKeyValueIterator rIter,
org.apache.hadoop.mapreduce.Counter inputKeyCounter,
org.apache.hadoop.mapreduce.Counter inputValueCounter,
- org.apache.hadoop.mapreduce.RecordWriter<OUTKEY,OUTVALUE> output,
+ org.apache.hadoop.mapreduce.RecordWriter<OUTKEY,OUTVALUE> output,
org.apache.hadoop.mapreduce.OutputCommitter committer,
org.apache.hadoop.mapreduce.StatusReporter reporter,
RawComparator<INKEY> comparator,
Class<INKEY> keyClass, Class<INVALUE> valueClass
) throws IOException, InterruptedException {
- RawKeyValueIterator r =
+ RawKeyValueIterator r =
new RawKeyValueIterator() {
-
+
@Override
public boolean next() throws IOException {
return rIter.next();
}
-
+
@Override
public DataInputBuffer getValue() throws IOException {
return rIter.getValue();
}
-
+
@Override
public Progress getProgress() {
return rIter.getProgress();
}
-
+
@Override
public DataInputBuffer getKey() throws IOException {
return rIter.getKey();
}
-
+
@Override
public void close() throws IOException {
rIter.close();
}
};
- org.apache.hadoop.mapreduce.ReduceContext<INKEY, INVALUE, OUTKEY, OUTVALUE>
- reduceContext =
+ org.apache.hadoop.mapreduce.ReduceContext<INKEY, INVALUE, OUTKEY, OUTVALUE>
+ reduceContext =
new ReduceContextImpl<INKEY, INVALUE, OUTKEY, OUTVALUE>(
- job,
- IDConverter.toMRTaskAttemptId(taskId),
- r,
- inputKeyCounter,
- inputValueCounter,
- output,
- committer,
- reporter,
- comparator,
- keyClass,
+ job,
+ IDConverter.toMRTaskAttemptId(taskId),
+ r,
+ inputKeyCounter,
+ inputValueCounter,
+ output,
+ committer,
+ reporter,
+ comparator,
+ keyClass,
valueClass);
- LOG.info("DEBUG: Using key class: " + keyClass + ", valueClass: " + valueClass);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using key class: " + keyClass
+ + ", valueClass: " + valueClass);
+ }
- org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE>.Context
- reducerContext =
+ org.apache.hadoop.mapreduce.Reducer<INKEY,INVALUE,OUTKEY,OUTVALUE>.Context
+ reducerContext =
new WrappedReducer<INKEY, INVALUE, OUTKEY, OUTVALUE>().getReducerContext(
reduceContext);
return reducerContext;
}
- public void taskCleanup(TezTaskUmbilicalProtocol umbilical)
+ public void taskCleanup(TezTaskUmbilicalProtocol umbilical)
throws IOException, InterruptedException {
// set phase for this task
setPhase(TezTaskStatus.Phase.CLEANUP);
@@ -693,13 +696,13 @@ extends RunningTaskContext {
committer.abortTask(taskAttemptContext);
}
- public void localizeConfiguration(JobConf jobConf)
+ public void localizeConfiguration(JobConf jobConf)
throws IOException, InterruptedException {
jobConf.set(JobContext.TASK_ID, IDConverter
.toMRTaskAttemptId(taskAttemptId).toString());
jobConf.set(JobContext.TASK_ATTEMPT_ID,
IDConverter.toMRTaskAttemptId(taskAttemptId).toString());
- jobConf.setInt(JobContext.TASK_PARTITION,
+ jobConf.setInt(JobContext.TASK_PARTITION,
taskAttemptId.getTaskID().getId());
jobConf.set(JobContext.ID, taskAttemptId.getTaskID().getVertexID().getDAGId().toString());
}
@@ -727,11 +730,11 @@ extends RunningTaskContext {
public JobContext getJobContext() {
return jobContext;
}
-
+
public TezTaskAttemptID getTaskAttemptId() {
return taskAttemptId;
}
-
+
public TezEngineTaskContext getTezEngineTaskContext() {
return tezEngineTaskContext;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index e2ef5b6..05b5411 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -322,12 +322,10 @@ public class YARNRunner implements ClientProtocol {
private TaskLocationHint[] getMapLocationHintsFromInputSplits(JobID jobId,
FileSystem fs, Configuration conf,
String jobSubmitDir) throws IOException {
- LOG.info("XXXX Reading splits information");
TaskSplitMetaInfo[] splitsInfo =
SplitMetaInfoReader.readSplitMetaInfo(jobId, fs, conf,
new Path(jobSubmitDir));
int splitsCount = splitsInfo.length;
- LOG.info("XXXX Found splits information, splitCount=" + splitsCount);
TaskLocationHint[] locationHints =
new TaskLocationHint[splitsCount];
for (int i = 0; i < splitsCount; ++i) {
@@ -462,11 +460,13 @@ public class YARNRunner implements ClientProtocol {
Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
getInitialClasspath(conf));
- LOG.info("XXXX Dumping out env for child, isMap=" + isMap);
- for (Map.Entry<String, String> entry : environment.entrySet()) {
- LOG.info("XXXX env entry: "
- + entry.getKey()
- + "=" + entry.getValue());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Dumping out env for child, isMap=" + isMap);
+ for (Map.Entry<String, String> entry : environment.entrySet()) {
+ LOG.debug("Child env entry: "
+ + entry.getKey()
+ + "=" + entry.getValue());
+ }
}
}
@@ -518,11 +518,13 @@ public class YARNRunner implements ClientProtocol {
jobLocalResources, i);
dag.addVertex(vertices[i]);
- LOG.info("XXXX Adding intermediate vertex to DAG"
- + ", vertexName=" + vertices[i].getVertexName()
- + ", processor=" + vertices[i].getProcessorName()
- + ", parrellism=" + vertices[i].getParallelism()
- + ", javaOpts=" + vertices[i].getJavaOpts());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding intermediate vertex to DAG"
+ + ", vertexName=" + vertices[i].getVertexName()
+ + ", processor=" + vertices[i].getProcessorName()
+ + ", parrellism=" + vertices[i].getParallelism()
+ + ", javaOpts=" + vertices[i].getJavaOpts());
+ }
}
return vertices;
}
@@ -540,10 +542,12 @@ public class YARNRunner implements ClientProtocol {
boolean isMRR = (intermediateReduces > 0);
- LOG.info("XXXX Parsing job config"
- + ", numMaps=" + numMaps
- + ", numReduces=" + numReduces
- + ", intermediateReduceStages=" + intermediateReduces);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Parsing job config"
+ + ", numMaps=" + numMaps
+ + ", numReduces=" + numReduces
+ + ", intermediateReduceStages=" + intermediateReduces);
+ }
// configure map vertex
String mapProcessor = "org.apache.tez.mapreduce.processor.map.MapProcessor";
@@ -575,11 +579,13 @@ public class YARNRunner implements ClientProtocol {
mapVertex.setJavaOpts(getMapJavaOpts(jobConf));
- LOG.info("XXXX Adding map vertex to DAG"
- + ", vertexName=" + mapVertex.getVertexName()
- + ", processor=" + mapVertex.getProcessorName()
- + ", parrellism=" + mapVertex.getParallelism()
- + ", javaOpts=" + mapVertex.getJavaOpts());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding map vertex to DAG"
+ + ", vertexName=" + mapVertex.getVertexName()
+ + ", processor=" + mapVertex.getProcessorName()
+ + ", parrellism=" + mapVertex.getParallelism()
+ + ", javaOpts=" + mapVertex.getJavaOpts());
+ }
dag.addVertex(mapVertex);
Vertex[] intermediateVertices = null;
@@ -618,11 +624,13 @@ public class YARNRunner implements ClientProtocol {
reduceVertex.setJavaOpts(getReduceJavaOpts(jobConf));
- LOG.info("XXXX Adding reduce vertex to DAG"
- + ", vertexName=" + reduceVertex.getVertexName()
- + ", processor=" + reduceVertex.getProcessorName()
- + ", parrellism=" + reduceVertex.getParallelism()
- + ", javaOpts=" + reduceVertex.getJavaOpts());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding reduce vertex to DAG"
+ + ", vertexName=" + reduceVertex.getVertexName()
+ + ", processor=" + reduceVertex.getProcessorName()
+ + ", parrellism=" + reduceVertex.getParallelism()
+ + ", javaOpts=" + reduceVertex.getJavaOpts());
+ }
dag.addVertex(reduceVertex);
EdgeProperty edgeProperty =
@@ -672,7 +680,9 @@ public class YARNRunner implements ClientProtocol {
Map<String, String> mrParamToDAGParamMap = DeprecatedKeys.getMRToDAGParamMap();
for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
if (mrConf.get(entry.getKey()) != null) {
- LOG.info("DEBUG: MR->DAG Setting new key: " + entry.getValue());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("MR->DAG Setting new key: " + entry.getValue());
+ }
dag.addConfiguration(entry.getValue(), mrConf.get(entry.getKey()));
}
}
[42/50] [abbrv] git commit: TEZ-26. Create tests for Vertex Scheduler
(bikas)
Posted by ss...@apache.org.
TEZ-26. Create tests for Vertex Scheduler (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a3e2c2f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a3e2c2f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a3e2c2f5
Branch: refs/heads/master
Commit: a3e2c2f57af79dc6d5b84b07c699e3d96a64758c
Parents: d94d37a
Author: Bikas Saha <bi...@apache.org>
Authored: Sat Jun 1 00:00:06 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Sat Jun 1 00:00:06 2013 -0700
----------------------------------------------------------------------
.../impl/BipartiteSlowStartVertexScheduler.java | 8 +-
.../tez/dag/app/dag/impl/TestDAGScheduler.java | 4 +-
.../tez/dag/app/dag/impl/TestVertexScheduler.java | 181 +++++++++++++++
3 files changed, 187 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a3e2c2f5/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
index 6c7bb0f..0321c10 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BipartiteSlowStartVertexScheduler.java
@@ -49,7 +49,6 @@ public class BipartiteSlowStartVertexScheduler implements VertexScheduler {
int numSourceTasks = 0;
int numSourceTasksCompleted = 0;
- boolean slowStartThresholdReached = false;
ArrayList<TezTaskID> pendingTasks;
int totalTasksToSchedule = 0;
HashMap<TezVertexID, Vertex> bipartiteSources =
@@ -84,7 +83,6 @@ public class BipartiteSlowStartVertexScheduler implements VertexScheduler {
@Override
public void onVertexStarted() {
- //targetVertex.scheduleTasks(targetVertex.getTasks().keySet());
pendingTasks = new ArrayList<TezTaskID>(managedVertex.getTotalTasks());
// track the tasks in this vertex
pendingTasks.addAll(managedVertex.getTasks().keySet());
@@ -140,7 +138,7 @@ public class BipartiteSlowStartVertexScheduler implements VertexScheduler {
}
float completedSourceTaskFraction = 0f;
- if (numSourceTasks != 0) {//support for 0 source tasks
+ if (numSourceTasks != 0) { // support for 0 source tasks
completedSourceTaskFraction = (float)numSourceTasksCompleted/numSourceTasks;
} else {
completedSourceTaskFraction = 1;
@@ -180,7 +178,9 @@ public class BipartiteSlowStartVertexScheduler implements VertexScheduler {
managedVertex.getVertexId() + " with totalTasks: " +
totalTasksToSchedule + ". " + numSourceTasksCompleted +
" source tasks completed out of " + numSourceTasks +
- ". SourceTaskCompletedFraction: " + completedSourceTaskFraction);
+ ". SourceTaskCompletedFraction: " + completedSourceTaskFraction +
+ " min: " + slowStartMinSrcCompletionFraction +
+ " max: " + slowStartMaxSrcCompletionFraction);
schedulePendingTasks(numTasksToSchedule);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a3e2c2f5/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
index 988a116..36c31a4 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java
@@ -45,7 +45,7 @@ public class TestDAGScheduler {
MockEventHandler mockEventHandler = new MockEventHandler();
- //@Test(timeout=10000)
+ @Test(timeout=10000)
public void testDAGSchedulerNaturalOrder() {
DAG mockDag = mock(DAG.class);
Vertex mockVertex = mock(Vertex.class);
@@ -72,7 +72,7 @@ public class TestDAGScheduler {
Assert.assertTrue(mockEventHandler.event.getPriority().getPriority() == 5);
}
- @Test//(timeout=10000)
+ @Test(timeout=10000)
public void testDAGSchedulerMRR() {
DAG mockDag = mock(DAG.class);
TezDAGID dagId = new TezDAGID("1", 1, 1);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a3e2c2f5/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexScheduler.java
new file mode 100644
index 0000000..3ebb653
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexScheduler.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+
+import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.app.dag.Task;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import static org.mockito.Mockito.*;
+
+public class TestVertexScheduler {
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Test
+ public void testBipartiteSlowStartVertexScheduler() {
+ BipartiteSlowStartVertexScheduler scheduler = null;
+ TezDAGID dagId = new TezDAGID("1", 1, 1);
+ HashMap<Vertex, EdgeProperty> mockInputVertices =
+ new HashMap<Vertex, EdgeProperty>();
+ Vertex mockSrcVertex1 = mock(Vertex.class);
+ TezVertexID mockSrcVertexId1 = new TezVertexID(dagId, 1);
+ EdgeProperty eProp1 = new EdgeProperty(
+ EdgeProperty.ConnectionPattern.BIPARTITE,
+ EdgeProperty.SourceType.STABLE, "in", "out");
+ when(mockSrcVertex1.getVertexId()).thenReturn(mockSrcVertexId1);
+ Vertex mockSrcVertex2 = mock(Vertex.class);
+ TezVertexID mockSrcVertexId2 = new TezVertexID(dagId, 2);
+ EdgeProperty eProp2 = new EdgeProperty(
+ EdgeProperty.ConnectionPattern.BIPARTITE,
+ EdgeProperty.SourceType.STABLE, "in", "out");
+ when(mockSrcVertex2.getVertexId()).thenReturn(mockSrcVertexId2);
+ Vertex mockSrcVertex3 = mock(Vertex.class);
+ TezVertexID mockSrcVertexId3 = new TezVertexID(dagId, 3);
+ EdgeProperty eProp3 = new EdgeProperty(
+ EdgeProperty.ConnectionPattern.ONE_TO_ALL,
+ EdgeProperty.SourceType.STABLE, "in", "out");
+ when(mockSrcVertex3.getVertexId()).thenReturn(mockSrcVertexId3);
+
+ Vertex mockManagedVertex = mock(Vertex.class);
+ TezVertexID mockManagedVertexId = new TezVertexID(dagId, 3);
+ when(mockManagedVertex.getVertexId()).thenReturn(mockManagedVertexId);
+ when(mockManagedVertex.getInputVertices()).thenReturn(mockInputVertices);
+
+ // fail if there is no bipartite src vertex
+ mockInputVertices.put(mockSrcVertex3, eProp3);
+ try {
+ scheduler = new BipartiteSlowStartVertexScheduler(mockManagedVertex, 0, 0);
+ Assert.assertFalse(true);
+ } catch (TezException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Atleast 1 bipartite source should exist"));
+ }
+
+ mockInputVertices.put(mockSrcVertex1, eProp1);
+ mockInputVertices.put(mockSrcVertex2, eProp2);
+
+ // check initialization
+ scheduler =
+ new BipartiteSlowStartVertexScheduler(mockManagedVertex, 0.1f, 0.1f);
+ Assert.assertTrue(scheduler.bipartiteSources.size() == 2);
+ Assert.assertTrue(scheduler.bipartiteSources.containsKey(mockSrcVertexId1));
+ Assert.assertTrue(scheduler.bipartiteSources.containsKey(mockSrcVertexId2));
+
+ HashMap<TezTaskID, Task> managedTasks = new HashMap<TezTaskID, Task>();
+ TezTaskID mockTaskId1 = new TezTaskID(mockManagedVertexId, 0);
+ managedTasks.put(mockTaskId1, null);
+ TezTaskID mockTaskId2 = new TezTaskID(mockManagedVertexId, 1);
+ managedTasks.put(mockTaskId2, null);
+ TezTaskID mockTaskId3 = new TezTaskID(mockManagedVertexId, 2);
+ managedTasks.put(mockTaskId3, null);
+
+ when(mockManagedVertex.getTotalTasks()).thenReturn(3);
+ when(mockManagedVertex.getTasks()).thenReturn(managedTasks);
+
+ final HashSet<TezTaskID> scheduledTasks = new HashSet<TezTaskID>();
+ doAnswer(new Answer() {
+ public Object answer(InvocationOnMock invocation) {
+ Object[] args = invocation.getArguments();
+ scheduledTasks.clear();
+ scheduledTasks.addAll((Collection<TezTaskID>)args[0]);
+ return null;
+ }}).when(mockManagedVertex).scheduleTasks(anyCollection());
+
+ // source vertices have 0 tasks. immediate start of all managed tasks
+ when(mockSrcVertex1.getTotalTasks()).thenReturn(0);
+ when(mockSrcVertex2.getTotalTasks()).thenReturn(0);
+ scheduler.onVertexStarted();
+ Assert.assertTrue(scheduler.pendingTasks.isEmpty());
+ Assert.assertTrue(scheduledTasks.size() == 3); // all tasks scheduled
+
+ when(mockSrcVertex1.getTotalTasks()).thenReturn(2);
+ when(mockSrcVertex2.getTotalTasks()).thenReturn(2);
+
+ // source vertex have some tasks. min, max == 0
+ scheduler = new BipartiteSlowStartVertexScheduler(mockManagedVertex, 0, 0);
+ scheduler.onVertexStarted();
+ Assert.assertTrue(scheduler.numSourceTasks == 4);
+ Assert.assertTrue(scheduler.totalTasksToSchedule == 3);
+ Assert.assertTrue(scheduler.numSourceTasksCompleted == 0);
+ Assert.assertTrue(scheduler.pendingTasks.isEmpty());
+ Assert.assertTrue(scheduledTasks.size() == 3); // all tasks scheduled
+
+ TezTaskAttemptID mockSrcAttemptId11 =
+ new TezTaskAttemptID(new TezTaskID(mockSrcVertexId1, 0), 0);
+ TezTaskAttemptID mockSrcAttemptId12 =
+ new TezTaskAttemptID(new TezTaskID(mockSrcVertexId1, 0), 1);
+ TezTaskAttemptID mockSrcAttemptId21 =
+ new TezTaskAttemptID(new TezTaskID(mockSrcVertexId2, 0), 0);
+ TezTaskAttemptID mockSrcAttemptId22 =
+ new TezTaskAttemptID(new TezTaskID(mockSrcVertexId2, 0), 1);
+ TezTaskAttemptID mockSrcAttemptId31 =
+ new TezTaskAttemptID(new TezTaskID(mockSrcVertexId3, 0), 0);
+
+ // min, max > 0 and min == max
+ scheduler =
+ new BipartiteSlowStartVertexScheduler(mockManagedVertex, 0.25f, 0.25f);
+ scheduler.onVertexStarted();
+ Assert.assertTrue(scheduler.pendingTasks.size() == 3); // no tasks scheduled
+ Assert.assertTrue(scheduler.numSourceTasks == 4);
+ // task completion from non-bipartite stage does nothing
+ scheduler.onSourceTaskCompleted(mockSrcAttemptId31);
+ Assert.assertTrue(scheduler.pendingTasks.size() == 3); // no tasks scheduled
+ Assert.assertTrue(scheduler.numSourceTasks == 4);
+ Assert.assertTrue(scheduler.numSourceTasksCompleted == 0);
+ scheduler.onSourceTaskCompleted(mockSrcAttemptId11);
+ Assert.assertTrue(scheduler.pendingTasks.isEmpty());
+ Assert.assertTrue(scheduledTasks.size() == 3); // all tasks scheduled
+ Assert.assertTrue(scheduler.numSourceTasksCompleted == 1);
+
+ // min, max > and min < max
+ scheduler =
+ new BipartiteSlowStartVertexScheduler(mockManagedVertex, 0.25f, 0.75f);
+ scheduler.onVertexStarted();
+ Assert.assertTrue(scheduler.pendingTasks.size() == 3); // no tasks scheduled
+ Assert.assertTrue(scheduler.numSourceTasks == 4);
+ scheduler.onSourceTaskCompleted(mockSrcAttemptId11);
+ scheduler.onSourceTaskCompleted(mockSrcAttemptId12);
+ Assert.assertTrue(scheduler.pendingTasks.size() == 2);
+ Assert.assertTrue(scheduledTasks.size() == 1); // 1 task scheduled
+ Assert.assertTrue(scheduler.numSourceTasksCompleted == 2);
+ scheduler.onSourceTaskCompleted(mockSrcAttemptId21);
+ Assert.assertTrue(scheduler.pendingTasks.size() == 0);
+ Assert.assertTrue(scheduledTasks.size() == 2); // 2 tasks scheduled
+ Assert.assertTrue(scheduler.numSourceTasksCompleted == 3);
+ scheduledTasks.clear();
+ scheduler.onSourceTaskCompleted(mockSrcAttemptId22); // we are done. no action
+ Assert.assertTrue(scheduler.pendingTasks.size() == 0);
+ Assert.assertTrue(scheduledTasks.size() == 0); // no task scheduled
+ Assert.assertTrue(scheduler.numSourceTasksCompleted == 4);
+
+ }
+}
[43/50] [abbrv] git commit: TEZ-23. TaskScheduler should depend on
official YARN AMRMClient* (bikas)
Posted by ss...@apache.org.
TEZ-23. TaskScheduler should depend on official YARN AMRMClient* (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/eeaaf232
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/eeaaf232
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/eeaaf232
Branch: refs/heads/master
Commit: eeaaf232f33a0b650ded7540c711d9d694e3157f
Parents: a3e2c2f
Author: Bikas Saha <bi...@apache.org>
Authored: Sat Jun 1 02:20:30 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Sat Jun 1 02:20:30 2013 -0700
----------------------------------------------------------------------
.../java/org/apache/tez/dag/app/rm/AMRMClient.java | 182 ------
.../org/apache/tez/dag/app/rm/AMRMClientAsync.java | 385 -------------
.../org/apache/tez/dag/app/rm/AMRMClientImpl.java | 447 ---------------
.../org/apache/tez/dag/app/rm/TaskScheduler.java | 91 ++--
.../tez/dag/app/rm/TaskSchedulerEventHandler.java | 5 +-
.../apache/tez/dag/app/rm/TestTaskScheduler.java | 103 +++-
6 files changed, 131 insertions(+), 1082 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClient.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClient.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClient.java
deleted file mode 100644
index 7e098f1..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClient.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.rm;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.service.Service;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public interface AMRMClient<T extends AMRMClient.ContainerRequest> extends Service {
-
- /**
- * Object to represent container request for resources.
- * Resources may be localized to nodes and racks.
- * Resources may be assigned priorities.
- * Can ask for multiple containers of a given type.
- */
- public static class ContainerRequest {
- Resource capability;
- String[] hosts;
- String[] racks;
- Priority priority;
- int containerCount;
-
- public ContainerRequest(Resource capability, String[] hosts,
- String[] racks, Priority priority, int containerCount) {
- this.capability = capability;
- this.hosts = (hosts != null ? hosts.clone() : null);
- this.racks = (racks != null ? racks.clone() : null);
- this.priority = priority;
- this.containerCount = containerCount;
- }
-
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("Capability[").append(capability).append("]");
- sb.append("Priority[").append(priority).append("]");
- sb.append("ContainerCount[").append(containerCount).append("]");
- return sb.toString();
- }
- }
-
- public static class StoredContainerRequest <T> extends ContainerRequest {
- T cookie;
-
- public StoredContainerRequest(Resource capability, String[] hosts,
- String[] racks, Priority priority) {
- super(capability, hosts, racks, priority, 1);
- }
-
- void setCookie(T cookie) {
- this.cookie = cookie;
- }
-
- T getCookie() {
- return cookie;
- }
- }
-
- /**
- * Register the application master. This must be called before any
- * other interaction
- * @param appHostName Name of the host on which master is running
- * @param appHostPort Port master is listening on
- * @param appTrackingUrl URL at which the master info can be seen
- * @return <code>RegisterApplicationMasterResponse</code>
- * @throws YarnRemoteException
- * @throws IOException
- */
- public RegisterApplicationMasterResponse
- registerApplicationMaster(String appHostName,
- int appHostPort,
- String appTrackingUrl)
- throws YarnRemoteException, IOException;
-
- /**
- * Request additional containers and receive new container allocations.
- * Requests made via <code>addContainerRequest</code> are sent to the
- * <code>ResourceManager</code>. New containers assigned to the master are
- * retrieved. Status of completed containers and node health updates are
- * also retrieved.
- * This also doubles up as a heartbeat to the ResourceManager and must be
- * made periodically.
- * The call may not always return any new allocations of containers.
- * App should not make concurrent allocate requests. May cause request loss.
- * @param progressIndicator Indicates progress made by the master
- * @return the response of the allocate request
- * @throws YarnRemoteException
- * @throws IOException
- */
- public AllocateResponse allocate(float progressIndicator)
- throws YarnRemoteException, IOException;
-
- /**
- * Unregister the application master. This must be called in the end.
- * @param appStatus Success/Failure status of the master
- * @param appMessage Diagnostics message on failure
- * @param appTrackingUrl New URL to get master info
- * @throws YarnRemoteException
- * @throws IOException
- */
- public void unregisterApplicationMaster(FinalApplicationStatus appStatus,
- String appMessage,
- String appTrackingUrl)
- throws YarnRemoteException, IOException;
-
- /**
- * Request containers for resources before calling <code>allocate</code>
- * @param req Resource request
- */
- public void addContainerRequest(T req);
-
- /**
- * Remove previous container request. The previous container request may have
- * already been sent to the ResourceManager. So even after the remove request
- * the app must be prepared to receive an allocation for the previous request
- * even after the remove request
- * @param req Resource request
- */
- public void removeContainerRequest(T req);
-
- /**
- * Release containers assigned by the Resource Manager. If the app cannot use
- * the container or wants to give up the container then it can release them.
- * The app needs to make new requests for the released resource capability if
- * it still needs it. eg. it released non-local resources
- * @param containerId
- */
- public void releaseAssignedContainer(ContainerId containerId);
-
- /**
- * Get the currently available resources in the cluster.
- * A valid value is available after a call to allocate has been made
- * @return Currently available resources
- */
- public Resource getClusterAvailableResources();
-
- /**
- * Get the current number of nodes in the cluster.
- * A valid values is available after a call to allocate has been made
- * @return Current number of nodes in the cluster
- */
- public int getClusterNodeCount();
-
- /**
- * Get outstanding <code>StoredContainerRequest</code>s matching the given
- * parameters. These StoredContainerRequests should have been added via
- * <code>addContainerRequest</code> earlier in the lifecycle.
- */
- public Collection<T> getMatchingRequests(
- Priority priority,
- String resourceName,
- Resource capability);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientAsync.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientAsync.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientAsync.java
deleted file mode 100644
index b38183c..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientAsync.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.rm;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.service.AbstractService;
-import org.apache.tez.dag.app.rm.AMRMClient.ContainerRequest;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * <code>AMRMClientAsync</code> handles communication with the ResourceManager
- * and provides asynchronous updates on events such as container allocations and
- * completions. It contains a thread that sends periodic heartbeats to the
- * ResourceManager.
- *
- * It should be used by implementing a CallbackHandler:
- * <pre>
- * {@code
- * class MyCallbackHandler implements AMRMClientAsync.CallbackHandler {
- * public void onContainersAllocated(List<Container> containers) {
- * [run tasks on the containers]
- * }
- *
- * public void onContainersCompleted(List<ContainerStatus> statuses) {
- * [update progress, check whether app is done]
- * }
- *
- * public void onNodesUpdated(List<NodeReport> updated) {}
- *
- * public void onReboot() {}
- * }
- * }
- * </pre>
- *
- * The client's lifecycle should be managed similarly to the following:
- *
- * <pre>
- * {@code
- * AMRMClientAsync asyncClient = new AMRMClientAsync(appAttId, 1000, new MyCallbackhandler());
- * asyncClient.init(conf);
- * asyncClient.start();
- * RegisterApplicationMasterResponse response = asyncClient
- * .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
- * appMasterTrackingUrl);
- * asyncClient.addContainerRequest(containerRequest);
- * [... wait for application to complete]
- * asyncClient.unregisterApplicationMaster(status, appMsg, trackingUrl);
- * asyncClient.stop();
- * }
- * </pre>
- */
-@Unstable
-@Evolving
-public class AMRMClientAsync<T extends ContainerRequest> extends AbstractService {
-
- private static final Log LOG = LogFactory.getLog(AMRMClientAsync.class);
-
- private final AMRMClient<T> client;
- private final int intervalMs;
- private final HeartbeatThread heartbeatThread;
- private final CallbackHandlerThread handlerThread;
- private final CallbackHandler handler;
-
- private final BlockingQueue<AllocateResponse> responseQueue;
-
- private volatile boolean keepRunning;
- private volatile float progress;
-
- private volatile Exception savedException;
-
- public AMRMClientAsync(ApplicationAttemptId id, int intervalMs,
- CallbackHandler callbackHandler) {
- this(new AMRMClientImpl<T>(id), intervalMs, callbackHandler);
- }
-
- @Private
- @VisibleForTesting
- public AMRMClientAsync(AMRMClient<T> client, int intervalMs,
- CallbackHandler callbackHandler) {
- super(AMRMClientAsync.class.getName());
- this.client = client;
- this.intervalMs = intervalMs;
- handler = callbackHandler;
- heartbeatThread = new HeartbeatThread();
- handlerThread = new CallbackHandlerThread();
- responseQueue = new LinkedBlockingQueue<AllocateResponse>();
- keepRunning = true;
- savedException = null;
- }
-
- @Override
- public void init(Configuration conf) {
- super.init(conf);
- client.init(conf);
- }
-
- @Override
- public void start() {
- handlerThread.start();
- client.start();
- super.start();
- }
-
- /**
- * Tells the heartbeat and handler threads to stop and waits for them to
- * terminate. Calling this method from the callback handler thread would cause
- * deadlock, and thus should be avoided.
- */
- @Override
- public void stop() {
- if (Thread.currentThread() == handlerThread) {
- throw new YarnException("Cannot call stop from callback handler thread!");
- }
- keepRunning = false;
- try {
- heartbeatThread.join();
- } catch (InterruptedException ex) {
- LOG.error("Error joining with heartbeat thread", ex);
- }
- client.stop();
- try {
- handlerThread.interrupt();
- handlerThread.join();
- } catch (InterruptedException ex) {
- LOG.error("Error joining with hander thread", ex);
- }
- super.stop();
- }
-
- public Collection<T> getMatchingRequests(
- Priority priority,
- String resourceName,
- Resource capability) {
- return client.getMatchingRequests(priority, resourceName, capability);
- }
-
- /**
- * Registers this application master with the resource manager. On successful
- * registration, starts the heartbeating thread.
- * @throws YarnRemoteException
- * @throws IOException
- */
- public RegisterApplicationMasterResponse registerApplicationMaster(
- String appHostName, int appHostPort, String appTrackingUrl)
- throws YarnRemoteException, IOException {
- RegisterApplicationMasterResponse response =
- client.registerApplicationMaster(appHostName, appHostPort, appTrackingUrl);
- heartbeatThread.start();
- return response;
- }
-
- /**
- * Unregister the application master. This must be called in the end.
- * @param appStatus Success/Failure status of the master
- * @param appMessage Diagnostics message on failure
- * @param appTrackingUrl New URL to get master info
- * @throws YarnRemoteException
- * @throws IOException
- */
- public void unregisterApplicationMaster(FinalApplicationStatus appStatus,
- String appMessage, String appTrackingUrl) throws YarnRemoteException, IOException {
- synchronized (client) {
- keepRunning = false;
- client.unregisterApplicationMaster(appStatus, appMessage, appTrackingUrl);
- }
- }
-
- /**
- * Request containers for resources before calling <code>allocate</code>
- * @param req Resource request
- */
- public void addContainerRequest(T req) {
- client.addContainerRequest(req);
- }
-
- /**
- * Remove previous container request. The previous container request may have
- * already been sent to the ResourceManager. So even after the remove request
- * the app must be prepared to receive an allocation for the previous request
- * even after the remove request
- * @param req Resource request
- */
- public void removeContainerRequest(T req) {
- client.removeContainerRequest(req);
- }
-
- /**
- * Release containers assigned by the Resource Manager. If the app cannot use
- * the container or wants to give up the container then it can release them.
- * The app needs to make new requests for the released resource capability if
- * it still needs it. eg. it released non-local resources
- * @param containerId
- */
- public void releaseAssignedContainer(ContainerId containerId) {
- client.releaseAssignedContainer(containerId);
- }
-
- /**
- * Get the currently available resources in the cluster.
- * A valid value is available after a call to allocate has been made
- * @return Currently available resources
- */
- public Resource getClusterAvailableResources() {
- return client.getClusterAvailableResources();
- }
-
- /**
- * Get the current number of nodes in the cluster.
- * A valid values is available after a call to allocate has been made
- * @return Current number of nodes in the cluster
- */
- public int getClusterNodeCount() {
- return client.getClusterNodeCount();
- }
-
- private class HeartbeatThread extends Thread {
- public HeartbeatThread() {
- super("AMRM Heartbeater thread");
- }
-
- public void run() {
- while (true) {
- AllocateResponse response = null;
- // synchronization ensures we don't send heartbeats after unregistering
- synchronized (client) {
- if (!keepRunning) {
- break;
- }
-
- try {
- response = client.allocate(progress);
- } catch (YarnRemoteException ex) {
- LOG.error("Yarn exception on heartbeat", ex);
- savedException = ex;
- // interrupt handler thread in case it waiting on the queue
- handlerThread.interrupt();
- break;
- } catch (IOException e) {
- LOG.error("IO exception on heartbeat", e);
- savedException = e;
- // interrupt handler thread in case it waiting on the queue
- handlerThread.interrupt();
- break;
- }
- }
- if (response != null) {
- while (true) {
- try {
- responseQueue.put(response);
- break;
- } catch (InterruptedException ex) {
- LOG.warn("Interrupted while waiting to put on response queue", ex);
- }
- }
- }
-
- try {
- Thread.sleep(intervalMs);
- } catch (InterruptedException ex) {
- LOG.warn("Heartbeater interrupted", ex);
- }
- }
- }
- }
-
- private class CallbackHandlerThread extends Thread {
- public CallbackHandlerThread() {
- super("AMRM Callback Handler Thread");
- }
-
- public void run() {
- while (keepRunning) {
- AllocateResponse response;
- try {
- if(savedException != null) {
- LOG.error("Stopping callback due to: ", savedException);
- handler.onError(savedException);
- break;
- }
- response = responseQueue.take();
- } catch (InterruptedException ex) {
- LOG.info("Interrupted while waiting for queue", ex);
- continue;
- }
-
- if (response.getReboot()) {
- handler.onRebootRequest();
- LOG.info("Reboot requested. Stopping callback.");
- break;
- }
- List<NodeReport> updatedNodes = response.getUpdatedNodes();
- if (!updatedNodes.isEmpty()) {
- handler.onNodesUpdated(updatedNodes);
- }
-
- List<ContainerStatus> completed =
- response.getCompletedContainersStatuses();
- if (!completed.isEmpty()) {
- handler.onContainersCompleted(completed);
- }
-
- List<Container> allocated = response.getAllocatedContainers();
- if (!allocated.isEmpty()) {
- handler.onContainersAllocated(allocated);
- }
-
- progress = handler.getProgress();
- }
- }
- }
-
- public interface CallbackHandler {
-
- /**
- * Called when the ResourceManager responds to a heartbeat with completed
- * containers. If the response contains both completed containers and
- * allocated containers, this will be called before containersAllocated.
- */
- public void onContainersCompleted(List<ContainerStatus> statuses);
-
- /**
- * Called when the ResourceManager responds to a heartbeat with allocated
- * containers. If the response containers both completed containers and
- * allocated containers, this will be called after containersCompleted.
- */
- public void onContainersAllocated(List<Container> containers);
-
- /**
- * Called when the ResourceManager wants the ApplicationMaster to reboot
- * for being out of sync. The ApplicationMaster should not unregister with
- * the RM unless the ApplicationMaster wants to be the last attempt.
- */
- public void onRebootRequest();
-
- /**
- * Called when nodes tracked by the ResourceManager have changed in health,
- * availability etc.
- */
- public void onNodesUpdated(List<NodeReport> updatedNodes);
-
- public float getProgress();
-
- public void onError(Exception e);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientImpl.java
deleted file mode 100644
index 75b9ad2..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMRMClientImpl.java
+++ /dev/null
@@ -1,447 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.dag.app.rm;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedAction;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.api.AMRMProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.service.AbstractService;
-import org.apache.hadoop.yarn.util.BuilderUtils;
-
-// TODO check inputs for null etc. YARN-654
-
-@Unstable
-public class AMRMClientImpl<T extends org.apache.tez.dag.app.rm.AMRMClient.ContainerRequest>
- extends AbstractService implements AMRMClient<T> {
-
- private static final Log LOG = LogFactory.getLog(AMRMClientImpl.class);
-
- private final RecordFactory recordFactory =
- RecordFactoryProvider.getRecordFactory(null);
-
- private int lastResponseId = 0;
-
- protected AMRMProtocol rmClient;
- protected final ApplicationAttemptId appAttemptId;
- protected Resource clusterAvailableResources;
- protected int clusterNodeCount;
-
- class ResourceRequestInfo {
- ResourceRequest remoteRequest;
- HashSet<T> containerRequests;
-
- ResourceRequestInfo(Priority priority, String resourceName, Resource capability) {
- remoteRequest = BuilderUtils.
- newResourceRequest(priority, resourceName, capability, 0);
- containerRequests = new HashSet<T>();
- }
- }
-
- //Key -> Priority
- //Value -> Map
- //Key->ResourceName (e.g., hostname, rackname, *)
- //Value->Map
- //Key->Resource Capability
- //Value->ResourceRequest
- protected final
- Map<Priority, Map<String, Map<Resource, ResourceRequestInfo>>>
- remoteRequestsTable =
- new TreeMap<Priority, Map<String, Map<Resource, ResourceRequestInfo>>>();
-
- protected final Set<ResourceRequest> ask = new TreeSet<ResourceRequest>(
- new org.apache.hadoop.yarn.util.BuilderUtils.ResourceRequestComparator());
- protected final Set<ContainerId> release = new TreeSet<ContainerId>();
-
- public AMRMClientImpl(ApplicationAttemptId appAttemptId) {
- super(AMRMClientImpl.class.getName());
- this.appAttemptId = appAttemptId;
- }
-
- @Override
- public synchronized void init(Configuration conf) {
- super.init(conf);
- }
-
- @Override
- public synchronized void start() {
- final YarnConfiguration conf = new YarnConfiguration(getConfig());
- final YarnRPC rpc = YarnRPC.create(conf);
- final InetSocketAddress rmAddress = conf.getSocketAddr(
- YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
-
- UserGroupInformation currentUser;
- try {
- currentUser = UserGroupInformation.getCurrentUser();
- } catch (IOException e) {
- throw new YarnException(e);
- }
-
- // CurrentUser should already have AMToken loaded.
- rmClient = currentUser.doAs(new PrivilegedAction<AMRMProtocol>() {
- @Override
- public AMRMProtocol run() {
- return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress,
- conf);
- }
- });
- LOG.debug("Connecting to ResourceManager at " + rmAddress);
- super.start();
- }
-
- @Override
- public synchronized void stop() {
- if (this.rmClient != null) {
- RPC.stopProxy(this.rmClient);
- }
- super.stop();
- }
-
- @Override
- public RegisterApplicationMasterResponse registerApplicationMaster(
- String appHostName, int appHostPort, String appTrackingUrl)
- throws YarnRemoteException, IOException {
- // do this only once ???
- RegisterApplicationMasterRequest request = recordFactory
- .newRecordInstance(RegisterApplicationMasterRequest.class);
- synchronized (this) {
- request.setApplicationAttemptId(appAttemptId);
- }
- request.setHost(appHostName);
- request.setRpcPort(appHostPort);
- if(appTrackingUrl != null) {
- request.setTrackingUrl(appTrackingUrl);
- }
- RegisterApplicationMasterResponse response = rmClient
- .registerApplicationMaster(request);
- return response;
- }
-
- @Override
- public AllocateResponse allocate(float progressIndicator)
- throws YarnRemoteException, IOException {
- AllocateResponse allocateResponse = null;
- ArrayList<ResourceRequest> askList = null;
- ArrayList<ContainerId> releaseList = null;
- AllocateRequest allocateRequest = null;
-
- try {
- synchronized (this) {
- askList = new ArrayList<ResourceRequest>(ask);
- releaseList = new ArrayList<ContainerId>(release);
- // optimistically clear this collection assuming no RPC failure
- ask.clear();
- release.clear();
- allocateRequest = BuilderUtils
- .newAllocateRequest(appAttemptId, lastResponseId, progressIndicator,
- askList, releaseList);
- }
-
- allocateResponse = rmClient.allocate(allocateRequest);
-
- synchronized (this) {
- // update these on successful RPC
- clusterNodeCount = allocateResponse.getNumClusterNodes();
- lastResponseId = allocateResponse.getResponseId();
- clusterAvailableResources = allocateResponse.getAvailableResources();
- }
- } finally {
- // TODO how to differentiate remote yarn exception vs error in rpc
- if(allocateResponse == null) {
- // we hit an exception in allocate()
- // preserve ask and release for next call to allocate()
- synchronized (this) {
- release.addAll(releaseList);
- // requests could have been added or deleted during call to allocate
- // If requests were added/removed then there is nothing to do since
- // the ResourceRequest object in ask would have the actual new value.
- // If ask does not have this ResourceRequest then it was unchanged and
- // so we can add the value back safely.
- // This assumes that there will no concurrent calls to allocate() and
- // so we dont have to worry about ask being changed in the
- // synchronized block at the beginning of this method.
- for(ResourceRequest oldAsk : askList) {
- if(!ask.contains(oldAsk)) {
- ask.add(oldAsk);
- }
- }
- }
- }
- }
- return allocateResponse;
- }
-
- @Override
- public void unregisterApplicationMaster(FinalApplicationStatus appStatus,
- String appMessage, String appTrackingUrl) throws YarnRemoteException,
- IOException {
- FinishApplicationMasterRequest request = recordFactory
- .newRecordInstance(FinishApplicationMasterRequest.class);
- request.setAppAttemptId(appAttemptId);
- request.setFinishApplicationStatus(appStatus);
- if(appMessage != null) {
- request.setDiagnostics(appMessage);
- }
- if(appTrackingUrl != null) {
- request.setTrackingUrl(appTrackingUrl);
- }
- rmClient.finishApplicationMaster(request);
- }
-
- @Override
- public synchronized void addContainerRequest(T req) {
- // Create resource requests
- // add check for dup locations
- if(req.hosts != null) {
- for (String host : req.hosts) {
- addResourceRequest(req.priority, host, req.capability, req.containerCount, req);
- }
- }
-
- if(req.racks != null) {
- for (String rack : req.racks) {
- addResourceRequest(req.priority, rack, req.capability, req.containerCount, req);
- }
- }
-
- // Off-switch
- addResourceRequest(req.priority, ResourceRequest.ANY, req.capability, req.containerCount, req);
- }
-
- @Override
- public synchronized void removeContainerRequest(T req) {
- // Update resource requests
- if(req.hosts != null) {
- for (String hostName : req.hosts) {
- decResourceRequest(req.priority, hostName, req.capability, req.containerCount, req);
- }
- }
-
- if(req.racks != null) {
- for (String rack : req.racks) {
- decResourceRequest(req.priority, rack, req.capability, req.containerCount, req);
- }
- }
-
- decResourceRequest(req.priority, ResourceRequest.ANY, req.capability, req.containerCount, req);
-
- }
-
- @Override
- public synchronized void releaseAssignedContainer(ContainerId containerId) {
- release.add(containerId);
- }
-
- @Override
- public synchronized Resource getClusterAvailableResources() {
- return clusterAvailableResources;
- }
-
- @Override
- public synchronized int getClusterNodeCount() {
- return clusterNodeCount;
- }
-
- @Override
- public synchronized Collection<T> getMatchingRequests(
- Priority priority,
- String resourceName,
- Resource capability) {
- Map<String, Map<Resource, ResourceRequestInfo>> remoteRequests =
- this.remoteRequestsTable.get(priority);
- if (remoteRequests == null) {
- return null;
- }
- Map<Resource, ResourceRequestInfo> reqMap = remoteRequests
- .get(resourceName);
- if (reqMap == null) {
- return null;
- }
- ResourceRequestInfo resourceRequestInfo = reqMap.get(capability);
- if (resourceRequestInfo == null) {
- return null;
- }
-
- return resourceRequestInfo.containerRequests;
- }
-
- private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
- // This code looks weird but is needed because of the following scenario.
- // A ResourceRequest is removed from the remoteRequestTable. A 0 container
- // request is added to 'ask' to notify the RM about not needing it any more.
- // Before the call to allocate, the user now requests more containers. If
- // the locations of the 0 size request and the new request are the same
- // (with the difference being only container count), then the set comparator
- // will consider both to be the same and not add the new request to ask. So
- // we need to check for the "same" request being present and remove it and
- // then add it back. The comparator is container count agnostic.
- // This should happen only rarely but we do need to guard against it.
- if(ask.contains(remoteRequest)) {
- ask.remove(remoteRequest);
- }
- ask.add(remoteRequest);
- }
-
- private void addResourceRequest(Priority priority, String resourceName,
- Resource capability, int containerCount, T req) {
- Map<String, Map<Resource, ResourceRequestInfo>> remoteRequests =
- this.remoteRequestsTable.get(priority);
- if (remoteRequests == null) {
- remoteRequests = new HashMap<String, Map<Resource, ResourceRequestInfo>>();
- this.remoteRequestsTable.put(priority, remoteRequests);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Added priority=" + priority);
- }
- }
- Map<Resource, ResourceRequestInfo> reqMap = remoteRequests.get(resourceName);
- if (reqMap == null) {
- reqMap = new HashMap<Resource, ResourceRequestInfo>();
- remoteRequests.put(resourceName, reqMap);
- }
- ResourceRequestInfo resourceRequestInfo = reqMap.get(capability);
- if (resourceRequestInfo == null) {
- resourceRequestInfo =
- new ResourceRequestInfo(priority, resourceName, capability);
- reqMap.put(capability, resourceRequestInfo);
- }
-
- resourceRequestInfo.remoteRequest.setNumContainers(
- resourceRequestInfo.remoteRequest.getNumContainers() + containerCount);
-
- if(req instanceof StoredContainerRequest<?>) {
- resourceRequestInfo.containerRequests.add(req);
- }
-
- // Note this down for next interaction with ResourceManager
- addResourceRequestToAsk(resourceRequestInfo.remoteRequest);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("addResourceRequest:" + " applicationId="
- + appAttemptId + " priority=" + priority.getPriority()
- + " resourceName=" + resourceName + " numContainers="
- + resourceRequestInfo.remoteRequest.getNumContainers()
- + " #asks=" + ask.size());
- }
- }
-
- private void decResourceRequest(Priority priority,
- String resourceName,
- Resource capability,
- int containerCount,
- T req) {
- Map<String, Map<Resource, ResourceRequestInfo>> remoteRequests =
- this.remoteRequestsTable.get(priority);
-
- if(remoteRequests == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Not decrementing resource as priority " + priority
- + " is not present in request table");
- }
- return;
- }
-
- Map<Resource, ResourceRequestInfo> reqMap = remoteRequests.get(resourceName);
- if (reqMap == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Not decrementing resource as " + resourceName
- + " is not present in request table");
- }
- return;
- }
- ResourceRequestInfo resourceRequestInfo = reqMap.get(capability);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("BEFORE decResourceRequest:" + " applicationId="
- + appAttemptId + " priority=" + priority.getPriority()
- + " resourceName=" + resourceName + " numContainers="
- + resourceRequestInfo.remoteRequest.getNumContainers()
- + " #asks=" + ask.size());
- }
-
- resourceRequestInfo.remoteRequest.setNumContainers(
- resourceRequestInfo.remoteRequest.getNumContainers() - containerCount);
-
- if(req instanceof StoredContainerRequest<?>) {
- resourceRequestInfo.containerRequests.remove(req);
- }
-
- if(resourceRequestInfo.remoteRequest.getNumContainers() < 0) {
- // guard against spurious removals
- resourceRequestInfo.remoteRequest.setNumContainers(0);
- }
- // send the ResourceRequest to RM even if is 0 because it needs to override
- // a previously sent value. If ResourceRequest was not sent previously then
- // sending 0 aught to be a no-op on RM
- addResourceRequestToAsk(resourceRequestInfo.remoteRequest);
-
- // delete entries from map if no longer needed
- if (resourceRequestInfo.remoteRequest.getNumContainers() == 0) {
- reqMap.remove(capability);
- if (reqMap.size() == 0) {
- remoteRequests.remove(resourceName);
- }
- if (remoteRequests.size() == 0) {
- remoteRequestsTable.remove(priority);
- }
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.info("AFTER decResourceRequest:" + " applicationId="
- + appAttemptId + " priority=" + priority.getPriority()
- + " resourceName=" + resourceName + " numContainers="
- + resourceRequestInfo.remoteRequest.getNumContainers()
- + " #asks=" + ask.size());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
index 14188b0..d081d2a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
@@ -42,11 +42,12 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.client.AMRMClient.StoredContainerRequest;
+import org.apache.hadoop.yarn.client.AMRMClientAsync;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.app.rm.AMRMClient.StoredContainerRequest;
import org.apache.tez.dag.app.rm.TaskScheduler.TaskSchedulerAppCallback.AppFinalStatus;
import com.google.common.annotations.VisibleForTesting;
@@ -98,11 +99,11 @@ public class TaskScheduler extends AbstractService
public AppFinalStatus getFinalAppStatus();
}
- final AMRMClientAsync<StoredContainerRequest<CRCookie>> amRmClient;
+ final AMRMClientAsync<CookieContainerRequest> amRmClient;
final TaskSchedulerAppCallback appClient;
- Map<Object, StoredContainerRequest<CRCookie>> taskRequests =
- new HashMap<Object, StoredContainerRequest<CRCookie>>();
+ Map<Object, CookieContainerRequest> taskRequests =
+ new HashMap<Object, CookieContainerRequest>();
Map<Object, Container> taskAllocations =
new HashMap<Object, Container>();
Map<ContainerId, Object> containerAssigments =
@@ -121,6 +122,19 @@ public class TaskScheduler extends AbstractService
Object appCookie;
}
+ class CookieContainerRequest extends StoredContainerRequest {
+ CRCookie cookie;
+ public CookieContainerRequest(Resource capability, String[] hosts,
+ String[] racks, Priority priority, CRCookie cookie) {
+ super(capability, hosts, racks, priority);
+ this.cookie = cookie;
+ }
+
+ CRCookie getCookie() {
+ return cookie;
+ }
+ }
+
public TaskScheduler(ApplicationAttemptId id,
TaskSchedulerAppCallback appClient,
String appHostName,
@@ -129,7 +143,7 @@ public class TaskScheduler extends AbstractService
super(TaskScheduler.class.getName());
this.appClient = appClient;
this.amRmClient =
- new AMRMClientAsync<StoredContainerRequest<CRCookie>>(id, 1000, this);
+ new AMRMClientAsync<CookieContainerRequest>(id, 1000, this);
this.appHostName = appHostName;
this.appHostPort = appHostPort;
this.appTrackingUrl = appTrackingUrl;
@@ -142,7 +156,7 @@ public class TaskScheduler extends AbstractService
String appHostName,
int appHostPort,
String appTrackingUrl,
- AMRMClientAsync<StoredContainerRequest<CRCookie>> client) {
+ AMRMClientAsync<CookieContainerRequest> client) {
super(TaskScheduler.class.getName());
this.appClient = appClient;
this.amRmClient = client;
@@ -270,13 +284,12 @@ public class TaskScheduler extends AbstractService
if(isStopped) {
return;
}
- Map<StoredContainerRequest<CRCookie>, Container> appContainers =
- new HashMap<StoredContainerRequest<CRCookie>, Container>
- (containers.size());
+ Map<CookieContainerRequest, Container> appContainers =
+ new HashMap<CookieContainerRequest, Container>(containers.size());
synchronized (this) {
for(Container container : containers) {
String location = container.getNodeId().getHost();
- StoredContainerRequest<CRCookie> assigned = getMatchingRequest(container, location);
+ CookieContainerRequest assigned = getMatchingRequest(container, location);
if(assigned == null) {
location = RackResolver.resolve(location).getNetworkLocation();
assigned = getMatchingRequest(container, location);
@@ -307,10 +320,10 @@ public class TaskScheduler extends AbstractService
}
// upcall to app must be outside locks
- for (Entry<StoredContainerRequest<CRCookie>, Container> entry :
- appContainers.entrySet()) {
- StoredContainerRequest<CRCookie> assigned = entry.getKey();
- appClient.taskAllocated(getTask(assigned), getAppCookie(assigned),
+ for (Entry<CookieContainerRequest, Container> entry :
+ appContainers.entrySet()) {
+ CookieContainerRequest assigned = entry.getKey();
+ appClient.taskAllocated(getTask(assigned), assigned.getCookie().appCookie,
entry.getValue());
}
}
@@ -357,16 +370,16 @@ public class TaskScheduler extends AbstractService
Priority priority,
Object clientCookie) {
// TODO check for nulls etc
- StoredContainerRequest<CRCookie> request =
- new StoredContainerRequest<CRCookie>(capability,
- hosts,
- racks,
- priority);
// TODO extra memory allocation
CRCookie cookie = new CRCookie();
cookie.task = task;
cookie.appCookie = clientCookie;
- request.setCookie(cookie);
+ CookieContainerRequest request =
+ new CookieContainerRequest(capability,
+ hosts,
+ racks,
+ priority,
+ cookie);
addTaskRequest(task, request);
LOG.info("Allocation request for task: " + task +
@@ -374,7 +387,7 @@ public class TaskScheduler extends AbstractService
}
public synchronized Container deallocateTask(Object task) {
- StoredContainerRequest<CRCookie> request = removeTaskRequest(task);
+ CookieContainerRequest request = removeTaskRequest(task);
if(request != null) {
// task not allocated yet
LOG.info("Deallocating task: " + task + " before allocation");
@@ -406,31 +419,29 @@ public class TaskScheduler extends AbstractService
return null;
}
- private StoredContainerRequest<CRCookie> getMatchingRequest(
+ private CookieContainerRequest getMatchingRequest(
Container container, String location) {
Priority priority = container.getPriority();
Resource capability = container.getResource();
- StoredContainerRequest<CRCookie> assigned = null;
- Collection<StoredContainerRequest<CRCookie>> requests =
+ CookieContainerRequest assigned = null;
+ List<? extends Collection<CookieContainerRequest>> requestsList =
amRmClient.getMatchingRequests(priority, location, capability);
- if(requests != null) {
- // TODO maybe do FIFO
- Iterator<StoredContainerRequest<CRCookie>> iterator = requests.iterator();
- if(iterator.hasNext()) {
- assigned = requests.iterator().next();
+ if(requestsList.size() > 0) {
+ // pick first one
+ for(Collection<CookieContainerRequest> requests : requestsList) {
+ Iterator<CookieContainerRequest> iterator = requests.iterator();
+ if(iterator.hasNext()) {
+ assigned = requests.iterator().next();
+ }
}
}
return assigned;
}
- private Object getTask(StoredContainerRequest<CRCookie> request) {
- return ((CRCookie)request.getCookie()).task;
- }
-
- private Object getAppCookie(StoredContainerRequest<CRCookie> request) {
- return ((CRCookie)request.getCookie()).appCookie;
+ private Object getTask(CookieContainerRequest request) {
+ return request.getCookie().task;
}
private void releaseContainer(ContainerId containerId, Object task) {
@@ -442,8 +453,8 @@ public class TaskScheduler extends AbstractService
private void assignContainer(Object task,
Container container,
- StoredContainerRequest<CRCookie> assigned) {
- StoredContainerRequest<CRCookie> request = removeTaskRequest(task);
+ CookieContainerRequest assigned) {
+ CookieContainerRequest request = removeTaskRequest(task);
assert request != null;
//assert assigned.equals(request);
@@ -453,8 +464,8 @@ public class TaskScheduler extends AbstractService
}
- private StoredContainerRequest<CRCookie> removeTaskRequest(Object task) {
- StoredContainerRequest<CRCookie> request = taskRequests.remove(task);
+ private CookieContainerRequest removeTaskRequest(Object task) {
+ CookieContainerRequest request = taskRequests.remove(task);
if(request != null) {
// remove all references of the request from AMRMClient
amRmClient.removeContainerRequest(request);
@@ -463,7 +474,7 @@ public class TaskScheduler extends AbstractService
}
private void addTaskRequest(Object task,
- StoredContainerRequest<CRCookie> request) {
+ CookieContainerRequest request) {
// TODO TEZ-37 fix duplicate handling
taskRequests.put(task, request);
amRmClient.addContainerRequest(request);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 8b98c68..e29db1c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -496,8 +496,6 @@ public class TaskSchedulerEventHandler extends AbstractService
// TODO TEZ-34 change event to reboot and send to app master
sendEvent(new DAGEvent(appContext.getDAGID(),
DAGEventType.INTERNAL_ERROR));
- throw new YarnException("ResourceManager requests reboot for: "
- + appContext.getApplicationID());
}
@Override
@@ -552,7 +550,8 @@ public class TaskSchedulerEventHandler extends AbstractService
@Override
public void onError(Exception e) {
- // TODO TEZ-35 handle error
+ sendEvent(new DAGEvent(appContext.getDAGID(),
+ DAGEventType.INTERNAL_ERROR));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/eeaaf232/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
index 4fecd3a..8d52e9a 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
@@ -19,6 +19,8 @@
package org.apache.tez.dag.app.rm;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -34,18 +36,20 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.client.AMRMClientAsync;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.RackResolver;
-import org.apache.tez.dag.app.rm.AMRMClient.StoredContainerRequest;
-import org.apache.tez.dag.app.rm.TaskScheduler.CRCookie;
+import org.apache.tez.dag.app.rm.TaskScheduler.CookieContainerRequest;
import org.apache.tez.dag.app.rm.TaskScheduler.TaskSchedulerAppCallback;
import org.apache.tez.dag.app.rm.TaskScheduler.TaskSchedulerAppCallback.AppFinalStatus;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
import static org.mockito.Mockito.*;
@@ -54,13 +58,13 @@ public class TestTaskScheduler {
RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
- @SuppressWarnings({ "unchecked", "rawtypes" })
+ @SuppressWarnings({ "unchecked" })
@Test
public void testTaskScheduler() throws Exception {
RackResolver.init(new YarnConfiguration());
TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class);
- AMRMClientAsync<StoredContainerRequest<CRCookie>> mockRMClient =
+ AMRMClientAsync<CookieContainerRequest> mockRMClient =
mock(AMRMClientAsync.class);
ApplicationAttemptId attemptId =
@@ -111,25 +115,25 @@ public class TestTaskScheduler {
String[] hosts = {"host1", "host5"};
String[] racks = {"/default-rack", "/default-rack"};
Priority mockPriority = mock(Priority.class);
- ArgumentCaptor<StoredContainerRequest> requestCaptor =
- ArgumentCaptor.forClass(StoredContainerRequest.class);
+ ArgumentCaptor<CookieContainerRequest> requestCaptor =
+ ArgumentCaptor.forClass(CookieContainerRequest.class);
// allocate task
scheduler.allocateTask(mockTask1, mockCapability, hosts,
racks, mockPriority, mockCookie1);
verify(mockRMClient, times(1)).
- addContainerRequest((StoredContainerRequest) any());
+ addContainerRequest((CookieContainerRequest) any());
// returned from task requests before allocation happens
Assert.assertNull(scheduler.deallocateTask(mockTask1));
verify(mockRMClient, times(1)).
- removeContainerRequest((StoredContainerRequest) any());
+ removeContainerRequest((CookieContainerRequest) any());
verify(mockRMClient, times(0)).
releaseAssignedContainer((ContainerId) any());
// deallocating unknown task
Assert.assertNull(scheduler.deallocateTask(mockTask1));
verify(mockRMClient, times(1)).
- removeContainerRequest((StoredContainerRequest) any());
+ removeContainerRequest((CookieContainerRequest) any());
verify(mockRMClient, times(0)).
releaseAssignedContainer((ContainerId) any());
@@ -142,17 +146,17 @@ public class TestTaskScheduler {
racks, mockPriority, mockCookie1);
verify(mockRMClient, times(2)).
addContainerRequest(requestCaptor.capture());
- StoredContainerRequest<CRCookie> request1 = requestCaptor.getValue();
+ CookieContainerRequest request1 = requestCaptor.getValue();
scheduler.allocateTask(mockTask2, mockCapability, hosts,
racks, mockPriority, mockCookie2);
verify(mockRMClient, times(3)).
addContainerRequest(requestCaptor.capture());
- StoredContainerRequest<CRCookie> request2 = requestCaptor.getValue();
+ CookieContainerRequest request2 = requestCaptor.getValue();
scheduler.allocateTask(mockTask3, mockCapability, hosts,
racks, mockPriority, mockCookie3);
verify(mockRMClient, times(4)).
addContainerRequest(requestCaptor.capture());
- StoredContainerRequest<CRCookie> request3 = requestCaptor.getValue();
+ CookieContainerRequest request3 = requestCaptor.getValue();
List<Container> containers = new ArrayList<Container>();
Container mockContainer1 = mock(Container.class, RETURNS_DEEP_STUBS);
@@ -175,35 +179,84 @@ public class TestTaskScheduler {
ContainerId mockCId4 = mock(ContainerId.class);
when(mockContainer4.getId()).thenReturn(mockCId4);
containers.add(mockContainer4);
- ArrayList<StoredContainerRequest<CRCookie>> hostContainers =
- new ArrayList<StoredContainerRequest<CRCookie>>();
+ ArrayList<CookieContainerRequest> hostContainers =
+ new ArrayList<CookieContainerRequest>();
hostContainers.add(request1);
hostContainers.add(request2);
hostContainers.add(request3);
- ArrayList<StoredContainerRequest<CRCookie>> rackContainers =
- new ArrayList<StoredContainerRequest<CRCookie>>();
+ ArrayList<CookieContainerRequest> rackContainers =
+ new ArrayList<CookieContainerRequest>();
rackContainers.add(request2);
rackContainers.add(request3);
- ArrayList<StoredContainerRequest<CRCookie>> anyContainers =
- new ArrayList<StoredContainerRequest<CRCookie>>();
+ ArrayList<CookieContainerRequest> anyContainers =
+ new ArrayList<CookieContainerRequest>();
anyContainers.add(request3);
+ final List<ArrayList<CookieContainerRequest>> hostList =
+ new LinkedList<ArrayList<CookieContainerRequest>>();
+ hostList.add(hostContainers);
+ final List<ArrayList<CookieContainerRequest>> rackList =
+ new LinkedList<ArrayList<CookieContainerRequest>>();
+ rackList.add(rackContainers);
+ final List<ArrayList<CookieContainerRequest>> anyList =
+ new LinkedList<ArrayList<CookieContainerRequest>>();
+ anyList.add(anyContainers);
+ final List<ArrayList<CookieContainerRequest>> emptyList =
+ new LinkedList<ArrayList<CookieContainerRequest>>();
// return all requests for host1
when(
mockRMClient.getMatchingRequests((Priority) any(), eq("host1"),
- (Resource) any())).thenReturn(hostContainers);
+ (Resource) any())).thenAnswer(
+ new Answer<List<? extends Collection<CookieContainerRequest>>>() {
+ @Override
+ public List<? extends Collection<CookieContainerRequest>> answer(
+ InvocationOnMock invocation) throws Throwable {
+ return hostList;
+ }
+
+ });
// first request matched by host
- // second request matched to rack. RackResolver by default puts hosts in
+ // second request matched to rack. RackResolver by default puts hosts in
// /default-rack. We need to workaround by returning rack matches only once
when(
- mockRMClient.getMatchingRequests((Priority) any(),
- eq("/default-rack"), (Resource) any())).thenReturn(
- rackContainers).thenReturn(null);
+ mockRMClient.getMatchingRequests((Priority) any(), eq("/default-rack"),
+ (Resource) any())).thenAnswer(
+ new Answer<List<? extends Collection<CookieContainerRequest>>>() {
+ @Override
+ public List<? extends Collection<CookieContainerRequest>> answer(
+ InvocationOnMock invocation) throws Throwable {
+ return rackList;
+ }
+
+ }).thenAnswer(
+ new Answer<List<? extends Collection<CookieContainerRequest>>>() {
+ @Override
+ public List<? extends Collection<CookieContainerRequest>> answer(
+ InvocationOnMock invocation) throws Throwable {
+ return emptyList;
+ }
+
+ });
// third request matched to ANY
when(
mockRMClient.getMatchingRequests((Priority) any(),
- eq(ResourceRequest.ANY), (Resource) any())).thenReturn(
- anyContainers).thenReturn(null);
+ eq(ResourceRequest.ANY), (Resource) any())).thenAnswer(
+ new Answer<List<? extends Collection<CookieContainerRequest>>>() {
+ @Override
+ public List<? extends Collection<CookieContainerRequest>> answer(
+ InvocationOnMock invocation) throws Throwable {
+ return anyList;
+ }
+
+ }).thenAnswer(
+ new Answer<List<? extends Collection<CookieContainerRequest>>>() {
+ @Override
+ public List<? extends Collection<CookieContainerRequest>> answer(
+ InvocationOnMock invocation) throws Throwable {
+ return emptyList;
+ }
+
+ });
scheduler.onContainersAllocated(containers);
// first container allocated
verify(mockApp).taskAllocated(mockTask1, mockCookie1, mockContainer1);
[36/50] [abbrv] git commit: TEZ-168. Fix synchronization issue in use
of container tokens when launching a container.
Posted by ss...@apache.org.
TEZ-168. Fix synchronization issue in use of container tokens when launching a container.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/34de34b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/34de34b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/34de34b3
Branch: refs/heads/master
Commit: 34de34b31ec87e7abbdd1fabe067801a4aa1fe01
Parents: 9f0d4c3
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu May 30 13:21:01 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu May 30 13:21:01 2013 -0700
----------------------------------------------------------------------
.../dag/app/rm/container/AMContainerHelpers.java | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/34de34b3/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 357eeda..c17f7f2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -188,7 +188,7 @@ public class AMContainerHelpers {
// Construct the actual Container
ContainerLaunchContext container =
ContainerLaunchContext.newInstance(lResources, myEnv, commands,
- myServiceData, commonContainerSpec.getTokens(), acls);
+ myServiceData, commonContainerSpec.getTokens().duplicate(), acls);
return container;
}
[05/50] [abbrv] git commit: TEZ-25: rename and cleanup: toplevel name
-> DAGProtos. Job message name -> DAGPlan
Posted by ss...@apache.org.
TEZ-25: rename and cleanup: toplevel name -> DAGProtos. Job message name -> DAGPlan
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/8aa2f300
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/8aa2f300
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/8aa2f300
Branch: refs/heads/master
Commit: 8aa2f300c80c9251f39db77830dd6a4bf7003794
Parents: 45db1be
Author: mikelid <mi...@microsoft.com>
Authored: Mon May 13 14:04:15 2013 -0700
Committer: mikelid <mi...@microsoft.com>
Committed: Mon May 13 14:04:15 2013 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/tez/dag/api/DAG.java | 15 +++++++++----
.../org/apache/tez/dag/api/DagTypeConverters.java | 16 +++++++++-----
tez-dag-api/src/main/proto/DAGPlan.proto | 4 +-
.../java/org/apache/tez/dag/api/TestDAGPlan.java | 15 ++++++++-----
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 16 +++++++-------
.../org/apache/tez/dag/app/MRRExampleHelper.java | 10 ++++----
.../main/java/org/apache/tez/dag/app/dag/DAG.java | 5 +--
.../java/org/apache/tez/dag/app/dag/Vertex.java | 2 +-
.../org/apache/tez/dag/app/dag/impl/DAGImpl.java | 12 +++++-----
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 2 +-
.../apache/tez/dag/utils/TezEngineChildJVM.java | 1 -
.../java/org/apache/tez/mapreduce/YARNRunner.java | 4 +-
12 files changed, 56 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
index efaf276..171921d 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -27,10 +27,15 @@ import java.util.Map.Entry;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.tez.dag.api.DAGPlan.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.DAGProtos.PlanKeyValuePair;
+import org.apache.tez.dag.api.DAGProtos.PlanLocalResource;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.DAGPlan.*;
-
public class DAG { // FIXME rename to Topology
List<Vertex> vertices;
@@ -88,8 +93,8 @@ public class DAG { // FIXME rename to Topology
}
// create protobuf message describing DAG
- public JobPlan createDag(){
- JobPlan.Builder jobBuilder = JobPlan.newBuilder();
+ public DAGPlan createDag(){
+ DAGPlan.Builder jobBuilder = DAGPlan.newBuilder();
jobBuilder.setName(this.name);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index afbabad..c5fd0a2 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -31,15 +31,19 @@ import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.DAGPlan.EdgePlan;
-import org.apache.tez.dag.api.DAGPlan.PlanKeyValuePair;
-import org.apache.tez.dag.api.DAGPlan.PlanLocalResource;
-import org.apache.tez.dag.api.DAGPlan.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGPlan.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.DAGProtos.PlanEdgeConnectionPattern;
+import org.apache.tez.dag.api.DAGProtos.PlanEdgeSourceType;
+import org.apache.tez.dag.api.DAGProtos.PlanKeyValuePair;
+import org.apache.tez.dag.api.DAGProtos.PlanLocalResource;
+import org.apache.tez.dag.api.DAGProtos.PlanLocalResourceType;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.PlanLocalResourceVisibility;
import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
import org.apache.tez.dag.api.EdgeProperty.SourceType;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.DAGPlan.*;
+
public class DagTypeConverters {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag-api/src/main/proto/DAGPlan.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGPlan.proto b/tez-dag-api/src/main/proto/DAGPlan.proto
index 38e8971..30fc54e 100644
--- a/tez-dag-api/src/main/proto/DAGPlan.proto
+++ b/tez-dag-api/src/main/proto/DAGPlan.proto
@@ -17,7 +17,7 @@
*/
option java_package = "org.apache.tez.dag.api";
-option java_outer_classname = "DAGPlan";
+option java_outer_classname = "DAGProtos";
option java_generate_equals_and_hash = true;
package org.apache.tez.dag.api;
@@ -111,7 +111,7 @@ message EdgePlan {
required string outputClass = 7;
}
-message JobPlan {
+message DAGPlan {
required string name = 1;
repeated VertexPlan vertex = 3;
repeated EdgePlan edge = 4;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
index f89a73e..a041f41 100644
--- a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
+++ b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -23,8 +23,11 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
-import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.DAGPlan.*;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -38,7 +41,7 @@ public class TestDAGPlan {
@Test
public void testBasicJobPlanSerde() throws IOException {
- JobPlan job = JobPlan.newBuilder()
+ DAGPlan job = DAGPlan.newBuilder()
.setName("test")
.addVertex(
VertexPlan.newBuilder()
@@ -46,7 +49,7 @@ public class TestDAGPlan {
.setType(PlanVertexType.NORMAL)
.addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("machineName").addRack("rack1").build())
.setTaskConfig(
- DAGPlan.PlanTaskConfiguration.newBuilder()
+ PlanTaskConfiguration.newBuilder()
.setNumTasks(2)
.setVirtualCores(4)
.setMemoryMb(1024)
@@ -67,11 +70,11 @@ public class TestDAGPlan {
}
}
- JobPlan inJob;
+ DAGPlan inJob;
FileInputStream inputStream;
try {
inputStream = new FileInputStream(file);
- inJob = JobPlan.newBuilder().mergeFrom(inputStream).build();
+ inJob = DAGPlan.newBuilder().mergeFrom(inputStream).build();
}
finally {
outStream.close();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 57003c1..174d210 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -63,8 +63,8 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.DAGPlan.JobPlan;
-import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.app.client.ClientService;
@@ -133,7 +133,7 @@ public class DAGAppMaster extends CompositeService {
public static final int SHUTDOWN_HOOK_PRIORITY = 30;
private Clock clock;
- private final JobPlan jobPlan;
+ private final DAGPlan jobPlan;
private long dagsStartTime;
private final long startTime;
private final long appSubmitTime;
@@ -180,14 +180,14 @@ public class DAGAppMaster extends CompositeService {
public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
- long appSubmitTime, JobPlan dagPB) {
+ long appSubmitTime, DAGPlan dagPB) {
this(applicationAttemptId, containerId, nmHost, nmPort, nmHttpPort,
new SystemClock(), appSubmitTime, dagPB);
}
public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
- Clock clock, long appSubmitTime, JobPlan dagPB) {
+ Clock clock, long appSubmitTime, DAGPlan dagPB) {
super(DAGAppMaster.class.getName());
this.jobPlan = dagPB;
this.clock = clock;
@@ -520,7 +520,7 @@ public class DAGAppMaster extends CompositeService {
// }
/** Create and initialize (but don't start) a single dag. */
- protected DAG createDAG(JobPlan dagPB) {
+ protected DAG createDAG(DAGPlan dagPB) {
// create single job
DAG newDag =
@@ -957,7 +957,7 @@ public class DAGAppMaster extends CompositeService {
String type;
TezConfiguration conf = new TezConfiguration(new YarnConfiguration());
- JobPlan jobPlan = null;
+ DAGPlan jobPlan = null;
if (cliParser.hasOption(OPT_PREDEFINED)) {
LOG.info("Running with PreDefined configuration");
type = cliParser.getOptionValue(OPT_PREDEFINED, "mr");
@@ -971,7 +971,7 @@ public class DAGAppMaster extends CompositeService {
}
else {
// Read the protobuf DAG
- JobPlan.Builder dagPlanBuilder = JobPlan.newBuilder();
+ DAGPlan.Builder dagPlanBuilder = DAGPlan.newBuilder();
FileInputStream dagPBBinaryStream = null;
try {
dagPBBinaryStream = new FileInputStream(TezConfiguration.DAG_AM_PLAN_PB_BINARY);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
index b50887a..0d845b2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
@@ -17,7 +17,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.dag.api.DAGPlan.JobPlan;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
@@ -87,7 +87,7 @@ public class MRRExampleHelper {
// TODO: these preconfigured jobs seem to require User and perhaps some other work.
// -> not tested with new DagPB system.
- static JobPlan createDAGConfigurationForMRR() throws IOException {
+ static DAGPlan createDAGConfigurationForMRR() throws IOException {
org.apache.tez.dag.api.DAG dag = new org.apache.tez.dag.api.DAG();
Vertex mapVertex = new Vertex("map",
"org.apache.tez.mapreduce.task.InitialTask", 6);
@@ -149,12 +149,12 @@ public class MRRExampleHelper {
dag.addConfiguration(MRJobConfig.MAP_SPECULATIVE, new Boolean(false).toString());
dag.addConfiguration(MRJobConfig.REDUCE_SPECULATIVE, new Boolean(false).toString());
- JobPlan dagPB = dag.createDag();
+ DAGPlan dagPB = dag.createDag();
return dagPB;
}
// TODO remove once client is in place
- static JobPlan createDAGConfigurationForMR() throws IOException {
+ static DAGPlan createDAGConfigurationForMR() throws IOException {
org.apache.tez.dag.api.DAG dag = new org.apache.tez.dag.api.DAG();
Vertex mapVertex = new Vertex("map",
"org.apache.tez.mapreduce.task.InitialTask", 6);
@@ -200,7 +200,7 @@ public class MRRExampleHelper {
dag.addConfiguration(MRJobConfig.MAP_SPECULATIVE, new Boolean(false).toString());
dag.addConfiguration(MRJobConfig.REDUCE_SPECULATIVE, new Boolean(false).toString());
- JobPlan dagPB = dag.createDag();
+ DAGPlan dagPB = dag.createDag();
return dagPB;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index 39cfc1a..da0e38a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -24,9 +24,8 @@ import java.util.Map;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGPlan.JobPlan;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.engine.records.TezDAGID;
import org.apache.tez.engine.records.TezVertexID;
@@ -60,7 +59,7 @@ public interface DAG {
TezConfiguration getConf();
- JobPlan getJobPlan();
+ DAGPlan getJobPlan();
/**
* @return the ACLs for this job for each type of JobACL given.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index 8d9fbc3..1e253aa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -28,7 +28,7 @@ import org.apache.tez.common.InputSpec;
import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.EdgeProperty;
-import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
import org.apache.tez.engine.records.TezTaskID;
import org.apache.tez.engine.records.TezVertexID;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 2e20692..cc3170e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -52,9 +52,9 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGPlan.EdgePlan;
-import org.apache.tez.dag.api.DAGPlan.JobPlan;
-import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint;
@@ -132,7 +132,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private TezCounters fullCounters = null;
public final TezConfiguration conf;
- private final JobPlan jobPlan;
+ private final DAGPlan jobPlan;
//fields initialized in init
private FileSystem fs;
@@ -311,7 +311,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
public DAGImpl(TezDAGID dagId, ApplicationAttemptId applicationAttemptId,
TezConfiguration conf,
- JobPlan jobPlan,
+ DAGPlan jobPlan,
EventHandler eventHandler,
TaskAttemptListener taskAttemptListener,
JobTokenSecretManager jobTokenSecretManager,
@@ -373,7 +373,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
@Override
- public JobPlan getJobPlan() {
+ public DAGPlan getJobPlan() {
return jobPlan;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index a423db1..b9e911c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -55,7 +55,7 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.tez.common.InputSpec;
import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
index b3db770..fee0846 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.app.rm.container.AMContainerHelpers;
import org.apache.tez.engine.records.TezVertexID;
public class TezEngineChildJVM {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8aa2f300/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 9042846..c0c9923 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -99,7 +99,7 @@ import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DAGPlan.JobPlan;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
@@ -804,7 +804,7 @@ public class YARNRunner implements ClientProtocol {
setDAGParamsFromMRConf(dag);
// emit protobuf DAG file style
- JobPlan dagPB = dag.createDag();
+ DAGPlan dagPB = dag.createDag();
FSDataOutputStream dagPBOutBinaryStream = null;
FSDataOutputStream dagPBOutTextStream = null;
Path binaryPath = new Path(jobSubmitDir, TezConfiguration.DAG_AM_PLAN_PB_BINARY);
[15/50] [abbrv] git commit: TEZ-9. Add support for client api and
records. cleaned up packages (bikas)
Posted by ss...@apache.org.
TEZ-9. Add support for client api and records. cleaned up packages (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/de5ee038
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/de5ee038
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/de5ee038
Branch: refs/heads/master
Commit: de5ee0382085e91e5644d881182309bc6858e9ed
Parents: cee79f6
Author: Bikas Saha <bi...@apache.org>
Authored: Fri May 17 15:59:37 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Fri May 17 15:59:37 2013 -0700
----------------------------------------------------------------------
.../java/org/apache/tez/dag/records/TezDAGID.java | 14 ++
tez-dag-api/pom.xml | 3 +-
.../src/main/java/org/apache/tez/dag/api/DAG.java | 16 +-
.../org/apache/tez/dag/api/DagTypeConverters.java | 18 +-
.../org/apache/tez/dag/api/client/DAGClient.java | 46 ++++
.../org/apache/tez/dag/api/client/DAGStatus.java | 95 ++++++++
.../org/apache/tez/dag/api/client/Progress.java | 51 +++++
.../apache/tez/dag/api/client/VertexStatus.java | 75 +++++++
.../client/rpc/DAGClientAMProtocolBlockingPB.java | 30 +++
.../tez/dag/api/client/rpc/DAGClientRPCImpl.java | 103 +++++++++
tez-dag-api/src/main/proto/DAGApiRecords.proto | 167 +++++++++++++++
.../src/main/proto/DAGClientAMProtocol.proto | 56 +++++
tez-dag-api/src/main/proto/DAGPlan.proto | 119 ----------
.../java/org/apache/tez/dag/api/TestDAGPlan.java | 10 +-
.../hadoop/mapred/MRVertexOutputCommitter.java | 8 +-
.../tez/dag/api/client/DAGStatusBuilder.java | 86 ++++++++
.../apache/tez/dag/api/client/ProgressBuilder.java | 58 +++++
.../tez/dag/api/client/VertexStatusBuilder.java | 78 +++++++
.../tez/dag/api/client/impl/TezBuilderUtils.java | 54 -----
.../apache/tez/dag/api/committer/DAGStatus.java | 39 ++++
.../api/committer/NullVertexOutputCommitter.java | 47 ++++
.../tez/dag/api/committer/VertexContext.java | 37 ++++
.../dag/api/committer/VertexOutputCommitter.java | 69 ++++++
.../apache/tez/dag/api/committer/VertexStatus.java | 51 +++++
.../org/apache/tez/dag/api/impl/DAGStatus.java | 39 ----
.../dag/api/impl/NullVertexOutputCommitter.java | 47 ----
.../org/apache/tez/dag/api/impl/VertexContext.java | 37 ----
.../tez/dag/api/impl/VertexOutputCommitter.java | 69 ------
.../org/apache/tez/dag/api/impl/VertexStatus.java | 51 -----
.../org/apache/tez/dag/api/oldrecords/AMInfo.java | 38 ++++
.../tez/dag/api/oldrecords/TaskAttemptReport.java | 66 ++++++
.../tez/dag/api/oldrecords/TaskAttemptState.java | 29 +++
.../apache/tez/dag/api/oldrecords/TaskReport.java | 63 ++++++
.../apache/tez/dag/api/oldrecords/TaskState.java | 23 ++
.../org/apache/tez/dag/api/records/AMInfo.java | 38 ----
.../tez/dag/api/records/TaskAttemptReport.java | 66 ------
.../tez/dag/api/records/TaskAttemptState.java | 29 ---
.../org/apache/tez/dag/api/records/TaskReport.java | 63 ------
.../org/apache/tez/dag/api/records/TaskState.java | 23 --
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 41 ++++-
.../org/apache/tez/dag/app/MRRExampleHelper.java | 2 +-
.../main/java/org/apache/tez/dag/app/dag/DAG.java | 7 +-
.../java/org/apache/tez/dag/app/dag/DAGReport.java | 2 +-
.../main/java/org/apache/tez/dag/app/dag/Task.java | 4 +-
.../org/apache/tez/dag/app/dag/TaskAttempt.java | 4 +-
.../java/org/apache/tez/dag/app/dag/Vertex.java | 6 +-
.../dag/event/TaskAttemptEventStatusUpdate.java | 2 +-
.../app/dag/event/VertexEventTaskCompleted.java | 2 +-
.../org/apache/tez/dag/app/dag/impl/DAGImpl.java | 58 +++++-
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 6 +-
.../org/apache/tez/dag/app/dag/impl/TaskImpl.java | 6 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 51 ++++-
.../tez/dag/app/rm/AMSchedulerEventTAEnded.java | 2 +-
.../tez/dag/app/rm/TaskSchedulerEventHandler.java | 2 +-
.../tez/dag/app/speculate/DefaultSpeculator.java | 4 +-
.../app/speculate/LegacyTaskRuntimeEstimator.java | 2 +-
.../tez/dag/app/speculate/StartEndTimesBase.java | 2 +-
.../tez/dag/history/events/DAGFinishedEvent.java | 2 +-
.../history/events/TaskAttemptFinishedEvent.java | 2 +-
.../tez/dag/history/events/TaskFinishedEvent.java | 2 +-
.../dag/history/events/VertexFinishedEvent.java | 2 +-
.../org/apache/tez/dag/utils/TezBuilderUtils.java | 54 +++++
.../tez/dag/app/dag/impl/TestTaskAttempt.java | 2 +-
.../apache/tez/dag/app/dag/impl/TestTaskImpl.java | 4 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 26 ++--
.../java/org/apache/tez/mapreduce/YARNRunner.java | 2 +-
66 files changed, 1587 insertions(+), 723 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
index e977dc0..733a908 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
@@ -100,6 +100,20 @@ public class TezDAGID extends TezID {
public String toString() {
return appendTo(new StringBuilder(DAG)).toString();
}
+
+ public static TezDAGID fromString(String dagId) {
+ try {
+ String[] split = dagId.split("_");
+ String rmId = split[1];
+ int appId = Integer.parseInt(split[2]);
+ int id;
+ id = idFormat.parse(split[3]).intValue();
+ return new TezDAGID(rmId, appId, id);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
@Override
public void readFields(DataInput in) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag-api/pom.xml b/tez-dag-api/pom.xml
index 2b6aebb..74b1284 100644
--- a/tez-dag-api/pom.xml
+++ b/tez-dag-api/pom.xml
@@ -64,7 +64,8 @@
<source>
<directory>${basedir}/src/main/proto</directory>
<includes>
- <include>DAGPlan.proto</include>
+ <include>DAGApiRecords.proto</include>
+ <include>DAGClientAMProtocol.proto</include>
</includes>
</source>
<output>${project.build.directory}/generated-sources/java</output>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 300ef31..7fa764a 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -28,14 +28,14 @@ import java.util.Stack;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.EdgePlan;
-import org.apache.tez.dag.api.DAGProtos.PlanKeyValuePair;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResource;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
-import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResource;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
public class DAG { // FIXME rename to Topology
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index c5fd0a2..ba42e23 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -31,15 +31,15 @@ import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.DAGProtos.EdgePlan;
-import org.apache.tez.dag.api.DAGProtos.PlanEdgeConnectionPattern;
-import org.apache.tez.dag.api.DAGProtos.PlanEdgeSourceType;
-import org.apache.tez.dag.api.DAGProtos.PlanKeyValuePair;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResource;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResourceType;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
-import org.apache.tez.dag.api.DAGProtos.PlanLocalResourceVisibility;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeConnectionPattern;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeSourceType;
+import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResource;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResourceType;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanLocalResourceVisibility;
import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
import org.apache.tez.dag.api.EdgeProperty.SourceType;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
new file mode 100644
index 0000000..477b017
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java
@@ -0,0 +1,46 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.TezException;
+
+/*
+ * Inteface class for monitoring the <code>DAG</code> running in a Tez DAG
+ * Application Master.
+ */
+public interface DAGClient {
+
+ /**
+ * Return the identifiers for all DAG's
+ */
+ List<String> getAllDAGs() throws TezException;
+
+ /**
+ * Get the status of a DAG
+ */
+ DAGStatus getDAGStatus(String dagId) throws TezException;
+
+ /**
+ * Get the status of a Vertex of a DAG
+ */
+ VertexStatus getVertexStatus(String dagId,
+ String vertexName) throws TezException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
new file mode 100644
index 0000000..597789b
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
@@ -0,0 +1,95 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusProtoOrBuilder;
+import org.apache.tez.dag.api.records.DAGProtos.StringProgressPairProto;
+import org.apache.tez.dag.api.TezException;
+
+public class DAGStatus {
+
+ public enum State {
+ SUBMITTED,
+ INITING,
+ RUNNING,
+ SUCCEEDED,
+ KILLED,
+ FAILED,
+ ERROR
+ };
+
+ DAGStatusProtoOrBuilder proxy = null;
+ Progress progress = null;
+ Map<String, Progress> vertexProgress = null;
+
+ public DAGStatus(DAGStatusProtoOrBuilder proxy) {
+ this.proxy = proxy;
+ }
+
+ public State getState() {
+ switch(proxy.getState()) {
+ case DAG_SUBMITTED:
+ return DAGStatus.State.SUBMITTED;
+ case DAG_INITING:
+ return DAGStatus.State.INITING;
+ case DAG_RUNNING:
+ return DAGStatus.State.RUNNING;
+ case DAG_SUCCEEDED:
+ return DAGStatus.State.SUCCEEDED;
+ case DAG_FAILED:
+ return DAGStatus.State.FAILED;
+ case DAG_KILLED:
+ return DAGStatus.State.KILLED;
+ case DAG_ERROR:
+ return DAGStatus.State.ERROR;
+ default:
+ throw new TezException("Unsupported value for DAGStatus.State : " +
+ proxy.getState());
+ }
+ }
+
+ public List<String> getDiagnostics() {
+ return proxy.getDiagnosticsList();
+ }
+
+ public Progress getDAGProgress() {
+ if(progress == null && proxy.hasDAGProgress()) {
+ progress = new Progress(proxy.getDAGProgress());
+ }
+ return progress;
+ }
+
+ public Map<String, Progress> getVertexProgress() {
+ if(vertexProgress == null) {
+ if(proxy.getVertexProgressList() != null) {
+ List<StringProgressPairProto> kvList = proxy.getVertexProgressList();
+ vertexProgress = new HashMap<String, Progress>(kvList.size());
+ for(StringProgressPairProto kv : kvList){
+ vertexProgress.put(kv.getKey(), new Progress(kv.getProgress()));
+ }
+ }
+ }
+ return vertexProgress;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
new file mode 100644
index 0000000..7158e30
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/Progress.java
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import org.apache.tez.dag.api.records.DAGProtos.ProgressProtoOrBuilder;
+
+public class Progress {
+
+ ProgressProtoOrBuilder proxy = null;
+
+ Progress(ProgressProtoOrBuilder proxy) {
+ this.proxy = proxy;
+ }
+
+ public int getTotalTaskCount() {
+ return proxy.getTotalTaskCount();
+ }
+
+ public int getSucceededTaskCount() {
+ return proxy.getSucceededTaskCount();
+ }
+
+ public int getRunningTaskCount() {
+ return proxy.getRunningTaskCount();
+ }
+
+ public int getFailedTaskCount() {
+ return proxy.getFailedTaskCount();
+ }
+
+ public int getKilledTaskCount() {
+ return proxy.getKilledTaskCount();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
new file mode 100644
index 0000000..33b4de3
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
@@ -0,0 +1,75 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProtoOrBuilder;
+import org.apache.tez.dag.api.TezException;
+
+public class VertexStatus {
+
+ public enum State {
+ INITED,
+ RUNNING,
+ SUCCEEDED,
+ KILLED,
+ FAILED,
+ ERROR
+ };
+
+ VertexStatusProtoOrBuilder proxy = null;
+ Progress progress = null;
+
+ public VertexStatus(VertexStatusProtoOrBuilder proxy) {
+ this.proxy = proxy;
+ }
+
+ public State getState() {
+ switch(proxy.getState()) {
+ case VERTEX_INITED:
+ return VertexStatus.State.INITED;
+ case VERTEX_RUNNING:
+ return VertexStatus.State.RUNNING;
+ case VERTEX_SUCCEEDED:
+ return VertexStatus.State.SUCCEEDED;
+ case VERTEX_FAILED:
+ return VertexStatus.State.FAILED;
+ case VERTEX_KILLED:
+ return VertexStatus.State.KILLED;
+ case VERTEX_ERROR:
+ return VertexStatus.State.ERROR;
+ default:
+ throw new TezException("Unsupported value for VertexStatus.State : " +
+ proxy.getState());
+ }
+ }
+
+ public List<String> getDiagnostics() {
+ return proxy.getDiagnosticsList();
+ }
+
+ public Progress getProgress() {
+ if(progress == null && proxy.hasProgress()) {
+ progress = new Progress(proxy.getProgress());
+ }
+ return progress;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
new file mode 100644
index 0000000..a1ee18f
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.rpc;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.DAGClientAMProtocol;
+
+@ProtocolInfo(
+ protocolName = "org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolPB",
+ protocolVersion = 1)
+public interface DAGClientAMProtocolBlockingPB
+ extends DAGClientAMProtocol.BlockingInterface {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
new file mode 100644
index 0000000..9238f63
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.rpc;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAllDAGsRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetDAGStatusRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetVertexStatusRequestProto;
+
+import com.google.protobuf.ServiceException;
+
+public class DAGClientRPCImpl implements DAGClient, Closeable {
+
+ private DAGClientAMProtocolBlockingPB proxy = null;
+
+ public DAGClientRPCImpl(long clientVersion,
+ InetSocketAddress addr,
+ TezConfiguration conf) throws IOException {
+ RPC.setProtocolEngine(conf,
+ DAGClientAMProtocolBlockingPB.class,
+ ProtobufRpcEngine.class);
+ proxy =
+ (DAGClientAMProtocolBlockingPB)
+ RPC.getProxy(DAGClientAMProtocolBlockingPB.class,
+ clientVersion,
+ addr,
+ conf);
+ }
+
+ @Override
+ public List<String> getAllDAGs() throws TezException {
+ GetAllDAGsRequestProto requestProto =
+ GetAllDAGsRequestProto.newBuilder().build();
+ try {
+ return proxy.getAllDAGs(null, requestProto).getDagIdList();
+ } catch (ServiceException e) {
+ throw new TezException(e);
+ }
+ }
+
+ @Override
+ public DAGStatus getDAGStatus(String dagId) throws TezException {
+ GetDAGStatusRequestProto requestProto =
+ GetDAGStatusRequestProto.newBuilder().setDagId(dagId).build();
+
+ try {
+ return new DAGStatus(
+ proxy.getDAGStatus(null, requestProto).getDagStatus());
+ } catch (ServiceException e) {
+ throw new TezException(e);
+ }
+ }
+
+ @Override
+ public VertexStatus getVertexStatus(String dagId, String vertexName)
+ throws TezException {
+ GetVertexStatusRequestProto requestProto =
+ GetVertexStatusRequestProto.newBuilder().
+ setDagId(dagId).setVertexName(vertexName).build();
+
+ try {
+ return new VertexStatus(
+ proxy.getVertexStatus(null, requestProto).getVertexStatus());
+ } catch (ServiceException e) {
+ throw new TezException(e);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (this.proxy != null) {
+ RPC.stopProxy(this.proxy);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/proto/DAGApiRecords.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGApiRecords.proto b/tez-dag-api/src/main/proto/DAGApiRecords.proto
new file mode 100644
index 0000000..242715b
--- /dev/null
+++ b/tez-dag-api/src/main/proto/DAGApiRecords.proto
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tez.dag.api.records";
+option java_outer_classname = "DAGProtos";
+option java_generate_equals_and_hash = true;
+
+// DAG plan messages
+
+// Many of these types have a dual in the Tez-api. To reduce confusion, these types have prefix or suffix
+// of "Plan" to indicate they are to be used in the dag-plan.
+// The big types use a suffix: JobPlan, VertexPlan, EdgePlan
+// --> these get more direct use in the runtime and the naming is natural.
+// The enums and utility types use prefix: PlanVertexType, PlanEdgeConnectionPaatern, etc
+// --> there is not great naming choice for these that avoids ambiguity, but this one seems acceptable.
+
+enum PlanVertexType {
+ INPUT = 0;
+ NORMAL = 1;
+ OUTPUT = 2;
+}
+
+enum PlanEdgeConnectionPattern {
+ ONE_TO_ONE = 0;
+ ONE_TO_ALL = 1;
+ BIPARTITE = 2;
+}
+
+enum PlanEdgeSourceType {
+ STABLE = 0;
+ STABLE_PERSISTED = 1;
+ STREAMING = 2;
+}
+
+message PlanKeyValuePair {
+ required string key = 1;
+ required string value = 2;
+}
+
+enum PlanLocalResourceType {
+ FILE = 0;
+ ARCHIVE = 1;
+ PATTERN = 2;
+}
+
+enum PlanLocalResourceVisibility {
+ PUBLIC = 0;
+ PRIVATE = 1;
+ APPLICATION = 2;
+}
+
+message PlanLocalResource {
+ required string name = 1;
+ required string uri = 2;
+ required int64 size = 3;
+ required int64 timeStamp = 4;
+ required PlanLocalResourceType type = 5;
+ required PlanLocalResourceVisibility visibility = 6;
+ optional string pattern = 7; // only used if type=PATTERN
+}
+
+// Each taskLocationHint represents a single split in in the input.
+// It is the list of [{rack,machines}] that host a replica of each particular split.
+// For now it is represented as pair-of-arrays rather than array-of-pairs.
+message PlanTaskLocationHint {
+ repeated string rack = 1;
+ repeated string host = 2;
+}
+
+message PlanTaskConfiguration {
+ required int32 numTasks = 1;
+ required int32 memoryMb = 2;
+ required int32 virtualCores = 3;
+ required string javaOpts = 4;
+ required string taskModule = 5;
+ repeated PlanLocalResource localResource = 6;
+ repeated PlanKeyValuePair environmentSetting = 8;
+}
+
+message VertexPlan {
+ required string name = 1;
+ required PlanVertexType type = 2;
+ optional string processorName = 3;
+ required PlanTaskConfiguration taskConfig = 4;
+ repeated PlanTaskLocationHint taskLocationHint = 7;
+ repeated string inEdgeId = 8;
+ repeated string outEdgeId = 9;
+}
+
+message EdgePlan {
+ required string id = 1;
+ required string inputVertexName = 2;
+ required string outputVertexName = 3;
+ required PlanEdgeConnectionPattern connectionPattern = 4;
+ required PlanEdgeSourceType sourceType = 5;
+ required string inputClass = 6;
+ required string outputClass = 7;
+}
+
+message DAGPlan {
+ required string name = 1;
+ repeated VertexPlan vertex = 2;
+ repeated EdgePlan edge = 3;
+ repeated PlanKeyValuePair jobSetting = 4;
+}
+
+
+// DAG monitoring messages
+message ProgressProto {
+ optional int32 totalTaskCount = 1;
+ optional int32 succeededTaskCount = 2;
+ optional int32 runningTaskCount = 3;
+ optional int32 failedTaskCount = 4;
+ optional int32 killedTaskCount = 5;
+}
+
+enum VertexStatusStateProto {
+ VERTEX_INITED = 0;
+ VERTEX_RUNNING = 1;
+ VERTEX_SUCCEEDED = 2;
+ VERTEX_KILLED = 3;
+ VERTEX_FAILED = 4;
+ VERTEX_ERROR = 5;
+}
+
+message VertexStatusProto {
+ optional VertexStatusStateProto state = 1;
+ repeated string diagnostics = 2;
+ optional ProgressProto progress = 3;
+}
+
+enum DAGStatusStateProto {
+ DAG_SUBMITTED = 0;
+ DAG_INITING = 1;
+ DAG_RUNNING = 2;
+ DAG_SUCCEEDED = 3;
+ DAG_KILLED = 4;
+ DAG_FAILED = 5;
+ DAG_ERROR = 6;
+}
+
+message StringProgressPairProto {
+ required string key = 1;
+ required ProgressProto progress = 2;
+}
+
+message DAGStatusProto {
+ optional DAGStatusStateProto state = 1;
+ repeated string diagnostics = 2;
+ optional ProgressProto DAGProgress = 3;
+ repeated StringProgressPairProto vertexProgress = 4;
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto b/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto
new file mode 100644
index 0000000..18247a1
--- /dev/null
+++ b/tez-dag-api/src/main/proto/DAGClientAMProtocol.proto
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tez.dag.api.client.rpc";
+option java_outer_classname = "DAGClientAMProtocolRPC";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+//import "DAGClientAMProtocolRecords.proto";
+
+import "DAGApiRecords.proto";
+
+message GetAllDAGsRequestProto {
+}
+
+message GetAllDAGsResponseProto {
+ repeated string dagId = 1;
+}
+
+message GetDAGStatusRequestProto {
+ optional string dagId = 1;
+}
+
+message GetDAGStatusResponseProto {
+ optional DAGStatusProto dagStatus = 1;
+}
+
+message GetVertexStatusRequestProto {
+ optional string dagId = 1;
+ optional string vertexName = 2;
+}
+
+message GetVertexStatusResponseProto {
+ optional VertexStatusProto vertexStatus = 1;
+}
+
+service DAGClientAMProtocol {
+ rpc getAllDAGs (GetAllDAGsRequestProto) returns (GetAllDAGsResponseProto);
+ rpc getDAGStatus (GetDAGStatusRequestProto) returns (GetDAGStatusResponseProto);
+ rpc getVertexStatus (GetVertexStatusRequestProto) returns (GetVertexStatusResponseProto);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/main/proto/DAGPlan.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGPlan.proto b/tez-dag-api/src/main/proto/DAGPlan.proto
deleted file mode 100644
index 30fc54e..0000000
--- a/tez-dag-api/src/main/proto/DAGPlan.proto
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.tez.dag.api";
-option java_outer_classname = "DAGProtos";
-option java_generate_equals_and_hash = true;
-package org.apache.tez.dag.api;
-
-// Many of these types have a dual in the Tez-api. To reduce confusion, these types have prefix or suffix
-// of "Plan" to indicate they are to be used in the dag-plan.
-// The big types use a suffix: JobPlan, VertexPlan, EdgePlan
-// --> these get more direct use in the runtime and the naming is natural.
-// The enums and utility types use prefix: PlanVertexType, PlanEdgeConnectionPaatern, etc
-// --> there is not great naming choice for these that avoids ambiguity, but this one seems acceptable.
-
-enum PlanVertexType {
- INPUT = 0;
- NORMAL = 1;
- OUTPUT = 2;
-}
-
-enum PlanEdgeConnectionPattern {
- ONE_TO_ONE = 0;
- ONE_TO_ALL = 1;
- BIPARTITE = 2;
-}
-
-enum PlanEdgeSourceType {
- STABLE = 0;
- STABLE_PERSISTED = 1;
- STREAMING = 2;
-}
-
-message PlanKeyValuePair {
- required string key = 1;
- required string value = 2;
-}
-
-enum PlanLocalResourceType {
- FILE = 0;
- ARCHIVE = 1;
- PATTERN = 2;
-}
-
-enum PlanLocalResourceVisibility {
- PUBLIC = 0;
- PRIVATE = 1;
- APPLICATION = 2;
-}
-
-message PlanLocalResource {
- required string name = 1;
- required string uri = 2;
- required int64 size = 3;
- required int64 timeStamp = 4;
- required PlanLocalResourceType type = 5;
- required PlanLocalResourceVisibility visibility = 6;
- optional string pattern = 7; // only used if type=PATTERN
-}
-
-// Each taskLocationHint represents a single split in in the input.
-// It is the list of [{rack,machines}] that host a replica of each particular split.
-// For now it is represented as pair-of-arrays rather than array-of-pairs.
-message PlanTaskLocationHint {
- repeated string rack = 1;
- repeated string host = 2;
-}
-
-message PlanTaskConfiguration {
- required int32 numTasks = 1;
- required int32 memoryMb = 2;
- required int32 virtualCores = 3;
- required string javaOpts = 4;
- required string taskModule = 5;
- repeated PlanLocalResource localResource = 6;
- repeated PlanKeyValuePair environmentSetting = 8;
-}
-
-message VertexPlan {
- required string name = 1;
- required PlanVertexType type = 2;
- optional string processorName = 3;
- required PlanTaskConfiguration taskConfig = 4;
- repeated PlanTaskLocationHint taskLocationHint = 7;
- repeated string inEdgeId = 8;
- repeated string outEdgeId = 9;
-}
-
-message EdgePlan {
- required string id = 1;
- required string inputVertexName = 2;
- required string outputVertexName = 3;
- required PlanEdgeConnectionPattern connectionPattern = 4;
- required PlanEdgeSourceType sourceType = 5;
- required string inputClass = 6;
- required string outputClass = 7;
-}
-
-message DAGPlan {
- required string name = 1;
- repeated VertexPlan vertex = 3;
- repeated EdgePlan edge = 4;
- repeated PlanKeyValuePair jobSetting = 5;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
index a041f41..6dca610 100644
--- a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
+++ b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -23,11 +23,11 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
-import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
index a3b72c7..7c886d6 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
@@ -33,12 +33,12 @@ import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.YarnException;
-import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
-import org.apache.tez.dag.api.impl.VertexContext;
-import org.apache.tez.dag.api.impl.VertexOutputCommitter;
-import org.apache.tez.dag.api.impl.VertexStatus;
+import org.apache.tez.dag.api.committer.VertexContext;
+import org.apache.tez.dag.api.committer.VertexOutputCommitter;
+import org.apache.tez.dag.api.committer.VertexStatus;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.utils.TezBuilderUtils;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
public class MRVertexOutputCommitter extends VertexOutputCommitter {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
new file mode 100644
index 0000000..553347f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java
@@ -0,0 +1,86 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusStateProto;
+import org.apache.tez.dag.api.records.DAGProtos.StringProgressPairProto;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusProto;
+import org.apache.tez.dag.api.records.DAGProtos.DAGStatusProto.Builder;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.app.dag.DAGState;
+
+public class DAGStatusBuilder extends DAGStatus {
+
+ public DAGStatusBuilder() {
+ super(DAGStatusProto.newBuilder());
+ }
+
+ public void setState(DAGState state) {
+ getBuilder().setState(getProtoState(state));
+ }
+
+ public void setDiagnostics(List<String> diagnostics) {
+ Builder builder = getBuilder();
+ builder.clearDiagnostics();
+ builder.addAllDiagnostics(diagnostics);
+ }
+
+ public void setDAGProgress(ProgressBuilder progress) {
+ getBuilder().setDAGProgress(progress.getProto());
+ }
+
+ public void addVertexProgress(String name, ProgressBuilder progress) {
+ StringProgressPairProto.Builder builder = StringProgressPairProto.newBuilder();
+ builder.setKey(name);
+ builder.setProgress(progress.getProto());
+ getBuilder().addVertexProgress(builder.build());
+ }
+
+ public DAGStatusProto getProto() {
+ return getBuilder().build();
+ }
+
+ private DAGStatusStateProto getProtoState(DAGState state) {
+ switch(state) {
+ case NEW:
+ case INITED:
+ return DAGStatusStateProto.DAG_INITING;
+ case RUNNING:
+ return DAGStatusStateProto.DAG_RUNNING;
+ case SUCCEEDED:
+ return DAGStatusStateProto.DAG_SUCCEEDED;
+ case FAILED:
+ return DAGStatusStateProto.DAG_FAILED;
+ case KILLED:
+ case KILL_WAIT:
+ return DAGStatusStateProto.DAG_KILLED;
+ case ERROR:
+ return DAGStatusStateProto.DAG_ERROR;
+ default:
+ throw new TezException("Unsupported value for DAGState : " + state);
+ }
+ }
+
+ private DAGStatusProto.Builder getBuilder() {
+ return (Builder) this.proxy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java
new file mode 100644
index 0000000..6cedb3f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java
@@ -0,0 +1,58 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import org.apache.tez.dag.api.records.DAGProtos.ProgressProto;
+import org.apache.tez.dag.api.records.DAGProtos.ProgressProto.Builder;
+import org.apache.tez.dag.api.client.Progress;
+
+public class ProgressBuilder extends Progress {
+
+ public ProgressBuilder() {
+ super(ProgressProto.newBuilder());
+ }
+
+ public ProgressProto getProto() {
+ return getBuilder().build();
+ }
+
+ public void setTotalTaskCount(int count) {
+ getBuilder().setTotalTaskCount(count);
+ }
+
+ public void setSucceededTaskCount(int count) {
+ getBuilder().setSucceededTaskCount(count);
+ }
+
+ public void setRunningTaskCount(int count) {
+ getBuilder().setRunningTaskCount(count);
+ }
+
+ public void setFailedTaskCount(int count) {
+ getBuilder().setFailedTaskCount(count);
+ }
+
+ public void setKilledTaskCount(int count) {
+ getBuilder().setKilledTaskCount(count);
+ }
+
+ private ProgressProto.Builder getBuilder() {
+ return (Builder) this.proxy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
new file mode 100644
index 0000000..91e3e83
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java
@@ -0,0 +1,78 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.api.client;
+
+import java.util.List;
+
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProto;
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProto.Builder;
+import org.apache.tez.dag.api.records.DAGProtos.VertexStatusStateProto;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.app.dag.VertexState;
+
+public class VertexStatusBuilder extends VertexStatus {
+
+ public VertexStatusBuilder() {
+ super(VertexStatusProto.newBuilder());
+ }
+
+ public void setState(VertexState state) {
+ getBuilder().setState(getProtoState(state));
+ }
+
+ public void setDiagnostics(List<String> diagnostics) {
+ Builder builder = getBuilder();
+ builder.clearDiagnostics();
+ builder.addAllDiagnostics(diagnostics);
+ }
+
+ public void setProgress(ProgressBuilder progress) {
+ getBuilder().setProgress(progress.getProto());
+ }
+
+ public VertexStatusProto getProto() {
+ return getBuilder().build();
+ }
+
+ private VertexStatusStateProto getProtoState(VertexState state) {
+ switch(state) {
+ case NEW:
+ case INITED:
+ return VertexStatusStateProto.VERTEX_INITED;
+ case RUNNING:
+ return VertexStatusStateProto.VERTEX_RUNNING;
+ case SUCCEEDED:
+ return VertexStatusStateProto.VERTEX_SUCCEEDED;
+ case FAILED:
+ return VertexStatusStateProto.VERTEX_FAILED;
+ case KILLED:
+ case KILL_WAIT:
+ return VertexStatusStateProto.VERTEX_KILLED;
+ case ERROR:
+ return VertexStatusStateProto.VERTEX_ERROR;
+ default:
+ throw new TezException("Unsupported value for VertexState : " + state);
+ }
+ }
+
+ private VertexStatusProto.Builder getBuilder() {
+ return (Builder) this.proxy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
deleted file mode 100644
index 5f36a58..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.client.impl;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.dag.api.records.AMInfo;
-import org.apache.tez.dag.app.dag.DAGReport;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-import org.apache.tez.dag.records.TezVertexID;
-
-public class TezBuilderUtils {
-
- public static TezVertexID newVertexID(TezDAGID dagId, int vertexId) {
- return new TezVertexID(dagId, vertexId);
- }
-
- public static TezTaskAttemptID newTaskAttemptId(TezTaskID taskId, int id) {
- return new TezTaskAttemptID(taskId, id);
- }
-
- public static DAGReport newDAGReport() {
- return null;
- }
-
- public static AMInfo newAMInfo(ApplicationAttemptId appAttemptID,
- long startTime, ContainerId containerID, String nmHost,
- int nmPort, int nmHttpPort) {
- return null;
- }
-
- public static TezTaskID newTaskId(TezDAGID dagId, int vertexId, int taskId) {
- return new TezTaskID(newVertexID(dagId, vertexId), taskId);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
new file mode 100644
index 0000000..587e18f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
@@ -0,0 +1,39 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.committer;
+
+public class DAGStatus {
+ public static enum State {
+ RUNNING(1),
+ SUCCEEDED(2),
+ FAILED(3),
+ PREP(4),
+ KILLED(5);
+
+ int value;
+
+ State(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return value;
+ }
+
+ };
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
new file mode 100644
index 0000000..c284ab5
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.committer;
+
+import java.io.IOException;
+
+import org.apache.tez.dag.api.committer.VertexStatus.State;
+
+public class NullVertexOutputCommitter extends VertexOutputCommitter {
+
+ @Override
+ public void init(VertexContext context) throws IOException {
+ // Nothing to do
+ }
+
+ @Override
+ public void setupVertex() throws IOException {
+ // Nothing to do
+ }
+
+ @Override
+ public void commitVertex() throws IOException {
+ // Nothing to do
+ }
+
+ @Override
+ public void abortVertex(State finalState) throws IOException {
+ // Nothing to do
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
new file mode 100644
index 0000000..14880d2
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.committer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
+
+public interface VertexContext {
+
+ public Configuration getConf();
+
+ public TezDAGID getDAGId();
+
+ // TODO Get rid of this as part of VertexContext cleanup.
+ public ApplicationAttemptId getApplicationAttemptId();
+
+ public TezVertexID getVertexId();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
new file mode 100644
index 0000000..b814d0e
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.committer;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Public
+@Unstable
+public abstract class VertexOutputCommitter {
+
+ /**
+ * Setup up the vertex output committer.
+ *
+ * @param context Context of the vertex whose output is being written.
+ * @throws IOException
+ */
+ public abstract void init(VertexContext context) throws IOException;
+
+ /**
+ * For the framework to setup the vertex output during initialization. This is
+ * called from the application master process for the vertex. This will be
+ * called multiple times, once per dag attempt for each vertex.
+ *
+ * @throws IOException if setup fails
+ */
+ public abstract void setupVertex() throws IOException;
+
+ /**
+ * For committing vertex's output after successful vertex completion.
+ * Note that this is invoked for vertices with a successful final state.
+ * This is called from the application master process for the entire vertex.
+ * This is guaranteed to only be called once.
+ * If it throws an exception the entire vertex will fail.
+ *
+ * @throws IOException
+ */
+ public abstract void commitVertex() throws IOException;
+
+ /**
+ * For aborting an unsuccessful vertex's output. Note that this is invoked for
+ * vertices with a final failed state. This is called from the application
+ * master process for the entire vertex. This may be called multiple times.
+ *
+ * @param state final runstate of the vertex
+ * @throws IOException
+ */
+ public abstract void abortVertex(VertexStatus.State finalState)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
new file mode 100644
index 0000000..a78027a
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.committer;
+
+public class VertexStatus {
+
+ public static enum State {
+ RUNNING(1),
+ SUCCEEDED(2),
+ FAILED(3),
+ PREP(4), // TODO change to INITING
+ KILLED(5);
+
+ int value;
+
+ State(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return value;
+ }
+
+ };
+
+ private State runState;
+
+ public synchronized State getState() {
+ return runState;
+ }
+
+ public synchronized void setState(State state) {
+ this.runState = state;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
deleted file mode 100644
index 6ede497..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.impl;
-
-public class DAGStatus {
- public static enum State {
- RUNNING(1),
- SUCCEEDED(2),
- FAILED(3),
- PREP(4),
- KILLED(5);
-
- int value;
-
- State(int value) {
- this.value = value;
- }
-
- public int getValue() {
- return value;
- }
-
- };
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
deleted file mode 100644
index 102bdec..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.impl;
-
-import java.io.IOException;
-
-import org.apache.tez.dag.api.impl.VertexStatus.State;
-
-public class NullVertexOutputCommitter extends VertexOutputCommitter {
-
- @Override
- public void init(VertexContext context) throws IOException {
- // Nothing to do
- }
-
- @Override
- public void setupVertex() throws IOException {
- // Nothing to do
- }
-
- @Override
- public void commitVertex() throws IOException {
- // Nothing to do
- }
-
- @Override
- public void abortVertex(State finalState) throws IOException {
- // Nothing to do
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
deleted file mode 100644
index 181aea5..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.impl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezVertexID;
-
-public interface VertexContext {
-
- public Configuration getConf();
-
- public TezDAGID getDAGId();
-
- // TODO Get rid of this as part of VertexContext cleanup.
- public ApplicationAttemptId getApplicationAttemptId();
-
- public TezVertexID getVertexId();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
deleted file mode 100644
index fe3e2d5..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.impl;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-@Public
-@Unstable
-public abstract class VertexOutputCommitter {
-
- /**
- * Setup up the vertex output committer.
- *
- * @param context Context of the vertex whose output is being written.
- * @throws IOException
- */
- public abstract void init(VertexContext context) throws IOException;
-
- /**
- * For the framework to setup the vertex output during initialization. This is
- * called from the application master process for the vertex. This will be
- * called multiple times, once per dag attempt for each vertex.
- *
- * @throws IOException if setup fails
- */
- public abstract void setupVertex() throws IOException;
-
- /**
- * For committing vertex's output after successful vertex completion.
- * Note that this is invoked for vertices with a successful final state.
- * This is called from the application master process for the entire vertex.
- * This is guaranteed to only be called once.
- * If it throws an exception the entire vertex will fail.
- *
- * @throws IOException
- */
- public abstract void commitVertex() throws IOException;
-
- /**
- * For aborting an unsuccessful vertex's output. Note that this is invoked for
- * vertices with a final failed state. This is called from the application
- * master process for the entire vertex. This may be called multiple times.
- *
- * @param state final runstate of the vertex
- * @throws IOException
- */
- public abstract void abortVertex(VertexStatus.State finalState)
- throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
deleted file mode 100644
index 9ba2ece..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.impl;
-
-public class VertexStatus {
-
- public static enum State {
- RUNNING(1),
- SUCCEEDED(2),
- FAILED(3),
- PREP(4),
- KILLED(5);
-
- int value;
-
- State(int value) {
- this.value = value;
- }
-
- public int getValue() {
- return value;
- }
-
- };
-
- private State runState;
-
- public synchronized State getState() {
- return runState;
- }
-
- public synchronized void setState(State state) {
- this.runState = state;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
new file mode 100644
index 0000000..7726d7b
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+public interface AMInfo {
+ public ApplicationAttemptId getAppAttemptId();
+ public long getStartTime();
+ public ContainerId getContainerId();
+ public String getNodeManagerHost();
+ public int getNodeManagerPort();
+ public int getNodeManagerHttpPort();
+
+ public void setAppAttemptId(ApplicationAttemptId appAttemptId);
+ public void setStartTime(long startTime);
+ public void setContainerId(ContainerId containerId);
+ public void setNodeManagerHost(String nmHost);
+ public void setNodeManagerPort(int nmPort);
+ public void setNodeManagerHttpPort(int mnHttpPort);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java
new file mode 100644
index 0000000..a2ee7e3
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+
+public interface TaskAttemptReport {
+ public abstract TezTaskAttemptID getTaskAttemptId();
+ public abstract TaskAttemptState getTaskAttemptState();
+ public abstract float getProgress();
+ public abstract long getStartTime();
+ public abstract long getFinishTime();
+ /** @return the shuffle finish time. Applicable only for reduce attempts */
+ public abstract long getShuffleFinishTime();
+ /** @return the sort/merge finish time. Applicable only for reduce attempts */
+ public abstract long getSortFinishTime();
+ public abstract TezCounters getCounters();
+ public abstract String getDiagnosticInfo();
+ public abstract String getStateString();
+ public abstract String getNodeManagerHost();
+ public abstract int getNodeManagerPort();
+ public abstract int getNodeManagerHttpPort();
+ public abstract ContainerId getContainerId();
+
+ public abstract void setTaskAttemptId(TezTaskAttemptID taskAttemptId);
+ public abstract void setTaskAttemptState(TaskAttemptState taskAttemptState);
+ public abstract void setProgress(float progress);
+ public abstract void setStartTime(long startTime);
+ public abstract void setFinishTime(long finishTime);
+ public abstract void setCounters(TezCounters counters);
+ public abstract void setDiagnosticInfo(String diagnosticInfo);
+ public abstract void setStateString(String stateString);
+ public abstract void setNodeManagerHost(String nmHost);
+ public abstract void setNodeManagerPort(int nmPort);
+ public abstract void setNodeManagerHttpPort(int nmHttpPort);
+ public abstract void setContainerId(ContainerId containerId);
+
+ /**
+ * Set the shuffle finish time. Applicable only for reduce attempts
+ * @param time the time the shuffle finished.
+ */
+ public abstract void setShuffleFinishTime(long time);
+ /**
+ * Set the sort/merge finish time. Applicable only for reduce attempts
+ * @param time the time the shuffle finished.
+ */
+ public abstract void setSortFinishTime(long time);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java
new file mode 100644
index 0000000..068913b
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+public enum TaskAttemptState {
+ NEW,
+ STARTING,
+ RUNNING,
+ COMMIT_PENDING,
+ SUCCEEDED,
+ FAILED,
+ KILLED
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java
new file mode 100644
index 0000000..3a0685f
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+import java.util.List;
+
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+
+public interface TaskReport {
+ public abstract TezTaskID getTaskId();
+ public abstract TaskState getTaskState();
+ public abstract float getProgress();
+ public abstract long getStartTime();
+ public abstract long getFinishTime();
+ public abstract TezCounters getCounters();
+
+ public abstract List<TezTaskAttemptID> getRunningAttemptsList();
+ public abstract TezTaskAttemptID getRunningAttempt(int index);
+ public abstract int getRunningAttemptsCount();
+
+ public abstract TezTaskAttemptID getSuccessfulAttempt();
+
+ public abstract List<String> getDiagnosticsList();
+ public abstract String getDiagnostics(int index);
+ public abstract int getDiagnosticsCount();
+
+
+ public abstract void setTaskId(TezTaskID taskId);
+ public abstract void setTaskState(TaskState taskState);
+ public abstract void setProgress(float progress);
+ public abstract void setStartTime(long startTime);
+ public abstract void setFinishTime(long finishTime);
+ public abstract void setCounters(TezCounters counters);
+
+ public abstract void addAllRunningAttempts(List<TezTaskAttemptID> taskAttempts);
+ public abstract void addRunningAttempt(TezTaskAttemptID taskAttempt);
+ public abstract void removeRunningAttempt(int index);
+ public abstract void clearRunningAttempts();
+
+ public abstract void setSuccessfulAttempt(TezTaskAttemptID taskAttempt);
+ public abstract void addAllDiagnostics(List<String> diagnostics);
+ public abstract void addDiagnostics(String diagnostics);
+ public abstract void removeDiagnostics(int index);
+ public abstract void clearDiagnostics();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java
new file mode 100644
index 0000000..c6b8767
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.oldrecords;
+
+public enum TaskState {
+ NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILLED
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java
deleted file mode 100644
index d57754a..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/AMInfo.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.records;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-
-public interface AMInfo {
- public ApplicationAttemptId getAppAttemptId();
- public long getStartTime();
- public ContainerId getContainerId();
- public String getNodeManagerHost();
- public int getNodeManagerPort();
- public int getNodeManagerHttpPort();
-
- public void setAppAttemptId(ApplicationAttemptId appAttemptId);
- public void setStartTime(long startTime);
- public void setContainerId(ContainerId containerId);
- public void setNodeManagerHost(String nmHost);
- public void setNodeManagerPort(int nmPort);
- public void setNodeManagerHttpPort(int mnHttpPort);
-}
\ No newline at end of file
[34/50] [abbrv] git commit: TEZ-113. Add unit tests for AMContainer.
(sseth)
Posted by ss...@apache.org.
TEZ-113. Add unit tests for AMContainer. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a74b4367
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a74b4367
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a74b4367
Branch: refs/heads/master
Commit: a74b43673dddf2f9d488f70b3bc337577b1c8dfa
Parents: 2340884
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed May 29 14:18:00 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed May 29 14:18:00 2013 -0700
----------------------------------------------------------------------
.../java/org/apache/tez/dag/app/AppContext.java | 2 +-
.../tez/dag/app/TaskAttemptListenerImpTezDag.java | 4 +-
.../dag/app/rm/container/AMContainerHelpers.java | 1 -
.../tez/dag/app/rm/container/AMContainerImpl.java | 19 +-
.../tez/dag/app/rm/container/TestAMContainer.java | 1008 +++++++++++++++
5 files changed, 1024 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a74b4367/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
index 041de56..03e1059 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
@@ -53,7 +53,7 @@ public interface AppContext {
long getStartTime();
- CharSequence getUser();
+ String getUser();
DAG getDAG();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a74b4367/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index d974cd8..7600db4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -504,9 +504,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
return COMPLETION_RESPONSE_NO_WAIT;
}
-
-
- // TODO EVENTUALLY remove all mrv2 ids.
+
@Override
public void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
attemptToContainerIdMap.remove(attemptId);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a74b4367/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index 6a9ae2e..357eeda 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -172,7 +172,6 @@ public class AMContainerHelpers {
Map<String, String> myEnv = new HashMap<String, String>(env.size());
myEnv.putAll(env);
myEnv.putAll(vertexEnv);
- // TODO TEZ-38 MRChildJVM2.setEnv should become a no-op
// Set up the launch command
List<String> commands = TezEngineChildJVM.getVMCommand(
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a74b4367/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 1089013..51246dd 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -54,8 +54,6 @@ import org.apache.tez.dag.app.rm.AMSchedulerEventContainerCompleted;
import org.apache.tez.dag.app.rm.AMSchedulerEventDeallocateContainer;
import org.apache.tez.dag.app.rm.NMCommunicatorLaunchRequestEvent;
import org.apache.tez.dag.app.rm.NMCommunicatorStopRequestEvent;
-import org.apache.tez.dag.history.DAGHistoryEvent;
-import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
import org.apache.tez.dag.records.TezTaskAttemptID;
@SuppressWarnings("rawtypes")
@@ -156,7 +154,7 @@ public class AMContainerImpl implements AMContainer {
.addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_TIMED_OUT, new TimedOutAtRunningTransition())
.addTransition(AMContainerState.RUNNING, AMContainerState.STOPPING, AMContainerEventType.C_NODE_FAILED, new NodeFailedAtRunningTransition())
.addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, AMContainerEventType.C_LAUNCHED, AMContainerEventType.C_LAUNCH_FAILED, AMContainerEventType.C_NM_STOP_SENT, AMContainerEventType.C_NM_STOP_FAILED), new ErrorAtRunningTransition())
-
+
.addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_ASSIGN_TA, new AssignTAAtWindDownTransition())
.addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_PULL_TA, new PullTAAfterStopTransition())
.addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.COMPLETED, AMContainerEventType.C_COMPLETED, new CompletedAtWindDownTransition())
@@ -168,11 +166,12 @@ public class AMContainerImpl implements AMContainer {
.addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_ASSIGN_TA, new AssignTAAtWindDownTransition())
.addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_PULL_TA, new PullTAAfterStopTransition())
+ // TODO This transition is wrong. Should be a noop / error.
.addTransition(AMContainerState.STOPPING, AMContainerState.COMPLETED, AMContainerEventType.C_COMPLETED, new CompletedAtWindDownTransition())
.addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_NODE_FAILED, new NodeFailedBaseTransition())
.addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, EnumSet.of(AMContainerEventType.C_LAUNCHED, AMContainerEventType.C_LAUNCH_FAILED, AMContainerEventType.C_TA_SUCCEEDED, AMContainerEventType.C_STOP_REQUEST, AMContainerEventType.C_NM_STOP_SENT, AMContainerEventType.C_NM_STOP_FAILED, AMContainerEventType.C_TIMED_OUT))
.addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_LAUNCH_REQUEST, new ErrorAtStoppingTransition())
-
+
.addTransition(AMContainerState.COMPLETED, AMContainerState.COMPLETED, AMContainerEventType.C_ASSIGN_TA, new AssignTAAtCompletedTransition())
.addTransition(AMContainerState.COMPLETED, AMContainerState.COMPLETED, AMContainerEventType.C_PULL_TA, new PullTAAfterStopTransition())
.addTransition(AMContainerState.COMPLETED, AMContainerState.COMPLETED, AMContainerEventType.C_NODE_FAILED, new NodeFailedBaseTransition())
@@ -236,7 +235,11 @@ public class AMContainerImpl implements AMContainer {
public List<TezTaskAttemptID> getQueuedTaskAttempts() {
readLock.lock();
try {
- return Collections.singletonList(this.pendingAttempt);
+ if (pendingAttempt != null) {
+ return Collections.singletonList(this.pendingAttempt);
+ } else {
+ return Collections.emptyList();
+ }
} finally {
readLock.unlock();
}
@@ -261,6 +264,10 @@ public class AMContainerImpl implements AMContainer {
readLock.unlock();
}
}
+
+ public boolean isInErrorState() {
+ return inError;
+ }
@Override
public void handle(AMContainerEvent event) {
@@ -333,6 +340,8 @@ public class AMContainerImpl implements AMContainer {
container.taskAttemptListener, event.getCredentials(),
event.shouldProfile(), container.appContext);
+ // Registering now, so that in case of delayed NM response, the child
+ // task is not told to die since the TAL does not know about the container.
container.registerWithTAListener();
container.sendStartRequestToNM();
LOG.info("Sending Launch Request for Container with id: " +
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a74b4367/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
new file mode 100644
index 0000000..574a2c5
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -0,0 +1,1008 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.rm.container;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.net.InetSocketAddress;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.tez.common.TezTaskContext;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.ContainerHeartbeatHandler;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventNodeFailed;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
+import org.apache.tez.dag.app.rm.AMSchedulerEventType;
+import org.apache.tez.dag.app.rm.NMCommunicatorEventType;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.engine.common.security.JobTokenIdentifier;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+public class TestAMContainer {
+
+
+ @Test
+ // Assign before launch.
+ public void tetSingleSuccessfulTaskFlow() {
+ WrappedContainer wc = new WrappedContainer();
+
+ wc.verifyState(AMContainerState.ALLOCATED);
+
+ // Launch request.
+ wc.launchContainer();
+ wc.verifyState(AMContainerState.LAUNCHING);
+ // 1 Launch request.
+ wc.verifyCountAndGetOutgoingEvents(1);
+
+ // Assign task.
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.verifyState(AMContainerState.LAUNCHING);
+ wc.verifyNoOutgoingEvents();
+ assertEquals(wc.taskAttemptID, wc.amContainer.getQueuedTaskAttempts()
+ .get(0));
+
+ // Container Launched
+ wc.containerLaunched();
+ wc.verifyState(AMContainerState.IDLE);
+ wc.verifyNoOutgoingEvents();
+ assertEquals(wc.taskAttemptID, wc.amContainer.getQueuedTaskAttempts()
+ .get(0));
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+ verify(wc.tal).registerRunningContainer(wc.containerID);
+ verify(wc.chh).register(wc.containerID);
+
+ // Pull TA
+ AMContainerTask pulledTask = wc.pullTaskToRun();
+ wc.verifyState(AMContainerState.RUNNING);
+ wc.verifyNoOutgoingEvents();
+ assertFalse(pulledTask.shouldDie());
+ assertEquals(wc.tezTaskContext.getTaskAttemptId(), pulledTask.getTask()
+ .getTaskAttemptId());
+ assertEquals(wc.taskAttemptID, wc.amContainer.getRunningTaskAttempt());
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+
+ wc.taskAttemptSucceeded(wc.taskAttemptID);
+ wc.verifyState(AMContainerState.IDLE);
+ wc.verifyNoOutgoingEvents();
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+ verify(wc.tal).unregisterTaskAttempt(wc.taskAttemptID);
+
+ wc.containerCompleted();
+ wc.verifyState(AMContainerState.COMPLETED);
+ // 1 Scheduler completed event.
+ wc.verifyCountAndGetOutgoingEvents(1);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.chh).unregister(wc.containerID);
+
+ assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+ assertFalse(wc.amContainer.isInErrorState());
+ }
+
+ @Test
+ // Assign after launch.
+ public void testSingleSuccessfulTaskFlow2() {
+ WrappedContainer wc = new WrappedContainer();
+
+ wc.verifyState(AMContainerState.ALLOCATED);
+
+ // Launch request.
+ wc.launchContainer();
+ wc.verifyState(AMContainerState.LAUNCHING);
+ // 1 Launch request.
+ wc.verifyCountAndGetOutgoingEvents(1);
+
+ // Container Launched
+ wc.containerLaunched();
+ wc.verifyState(AMContainerState.IDLE);
+ wc.verifyNoOutgoingEvents();
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+ verify(wc.tal).registerRunningContainer(wc.containerID);
+ verify(wc.chh).register(wc.containerID);
+
+ // Assign task.
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.verifyState(AMContainerState.IDLE);
+ wc.verifyNoOutgoingEvents();
+ assertEquals(wc.taskAttemptID, wc.amContainer.getQueuedTaskAttempts()
+ .get(0));
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+
+ // Pull TA
+ AMContainerTask pulledTask = wc.pullTaskToRun();
+ wc.verifyState(AMContainerState.RUNNING);
+ wc.verifyNoOutgoingEvents();
+ assertFalse(pulledTask.shouldDie());
+ assertEquals(wc.tezTaskContext.getTaskAttemptId(), pulledTask.getTask()
+ .getTaskAttemptId());
+ assertEquals(wc.taskAttemptID, wc.amContainer.getRunningTaskAttempt());
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+
+ wc.taskAttemptSucceeded(wc.taskAttemptID);
+ wc.verifyState(AMContainerState.IDLE);
+ wc.verifyNoOutgoingEvents();
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+ verify(wc.tal).unregisterTaskAttempt(wc.taskAttemptID);
+
+ wc.containerCompleted();
+ wc.verifyState(AMContainerState.COMPLETED);
+ // 1 Scheduler completed event.
+ wc.verifyCountAndGetOutgoingEvents(1);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.chh).unregister(wc.containerID);
+
+ assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+ assertFalse(wc.amContainer.isInErrorState());
+ }
+
+ @Test
+ public void testSingleSuccessfulTaskFlowStopRequest() {
+ WrappedContainer wc = new WrappedContainer();
+
+ wc.verifyState(AMContainerState.ALLOCATED);
+
+ wc.launchContainer();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.containerLaunched();
+ wc.pullTaskToRun();
+ wc.taskAttemptSucceeded(wc.taskAttemptID);
+
+ wc.stopRequest();
+ wc.verifyState(AMContainerState.STOP_REQUESTED);
+ // Event to NM to stop the container.
+ wc.verifyCountAndGetOutgoingEvents(1);
+ assertTrue(wc.verifyCountAndGetOutgoingEvents(1).get(0).getType() ==
+ NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+
+ wc.nmStopSent();
+ wc.verifyState(AMContainerState.STOPPING);
+ wc.verifyNoOutgoingEvents();
+
+ wc.containerCompleted();
+ wc.verifyState(AMContainerState.COMPLETED);
+ // 1 Scheduler completed event.
+ wc.verifyCountAndGetOutgoingEvents(1);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.chh).unregister(wc.containerID);
+
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+ assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+ assertFalse(wc.amContainer.isInErrorState());
+ }
+
+ @Test
+ public void testSingleSuccessfulTaskFlowFailedNMStopRequest() {
+ WrappedContainer wc = new WrappedContainer();
+
+ wc.verifyState(AMContainerState.ALLOCATED);
+
+ wc.launchContainer();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.containerLaunched();
+ wc.pullTaskToRun();
+ wc.taskAttemptSucceeded(wc.taskAttemptID);
+
+ wc.stopRequest();
+ wc.verifyState(AMContainerState.STOP_REQUESTED);
+ // Event to NM to stop the container.
+ wc.verifyCountAndGetOutgoingEvents(1);
+ assertTrue(wc.verifyCountAndGetOutgoingEvents(1).get(0).getType() ==
+ NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+
+ wc.nmStopFailed();
+ wc.verifyState(AMContainerState.STOPPING);
+ // Event to ask a RM container release.
+ wc.verifyCountAndGetOutgoingEvents(1);
+ assertTrue(wc.verifyCountAndGetOutgoingEvents(1).get(0).getType() ==
+ AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
+
+ wc.containerCompleted();
+ wc.verifyState(AMContainerState.COMPLETED);
+ // 1 Scheduler completed event.
+ wc.verifyCountAndGetOutgoingEvents(1);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.chh).unregister(wc.containerID);
+
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+ assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size());
+ assertFalse(wc.amContainer.isInErrorState());
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testMultipleAllocationsAtIdle() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+ wc.containerLaunched();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.verifyState(AMContainerState.IDLE);
+
+ TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+ wc.assignTaskAttempt(taID2);
+
+ wc.verifyState(AMContainerState.STOP_REQUESTED);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.chh).unregister(wc.containerID);
+ // 1 for NM stop request. 2 TERMINATING to TaskAttempt.
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATING);
+ assertTrue(wc.amContainer.isInErrorState());
+
+ wc.nmStopSent();
+ wc.containerCompleted();
+ // 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+// assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
+// assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testAllocationAtRunning() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+ wc.containerLaunched();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.pullTaskToRun();
+ wc.verifyState(AMContainerState.RUNNING);
+
+ TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+ wc.assignTaskAttempt(taID2);
+
+ wc.verifyState(AMContainerState.STOP_REQUESTED);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.chh).unregister(wc.containerID);
+ // 1 for NM stop request. 2 TERMINATING to TaskAttempt.
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATING);
+ assertTrue(wc.amContainer.isInErrorState());
+
+ wc.nmStopSent();
+ wc.containerCompleted();
+ // 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+// assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
+// assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
+// assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testMultipleAllocationsAtLaunching() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.pullTaskToRun();
+ wc.verifyState(AMContainerState.LAUNCHING);
+
+ TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+ wc.assignTaskAttempt(taID2);
+
+ wc.verifyState(AMContainerState.STOP_REQUESTED);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.chh).unregister(wc.containerID);
+ // 1 for NM stop request. 2 TERMINATING to TaskAttempt.
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATING);
+ assertTrue(wc.amContainer.isInErrorState());
+
+ wc.nmStopSent();
+ wc.containerCompleted();
+ // 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+// assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
+// assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
+// assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testContainerTimedOutAtRunning() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+ wc.containerLaunched();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.pullTaskToRun();
+ wc.verifyState(AMContainerState.RUNNING);
+
+ wc.containerTimedOut();
+ wc.verifyState(AMContainerState.STOP_REQUESTED);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.chh).unregister(wc.containerID);
+ // 1 to TA, 1 for RM de-allocate.
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+ NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
+ // TODO Should this be an RM DE-ALLOCATE instead ?
+
+ wc.containerCompleted();
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+ assertFalse(wc.amContainer.isInErrorState());
+
+// assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO Set/Unset properly.
+// assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
+// assertEquals(1, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testLaunchFailure() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.verifyState(AMContainerState.LAUNCHING);
+ wc.launchFailed();
+ wc.verifyState(AMContainerState.STOPPING);
+ verify(wc.tal).registerRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+ AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
+
+ wc.containerCompleted();
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+ // Valid transition. Container complete, but not with an error.
+ assertFalse(wc.amContainer.isInErrorState());
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testContainerCompletedAtAllocated() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+ wc.verifyState(AMContainerState.ALLOCATED);
+
+ wc.containerCompleted();
+ wc.verifyState(AMContainerState.COMPLETED);
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+ assertFalse(wc.amContainer.isInErrorState());
+ }
+
+ @Ignore
+ @SuppressWarnings("rawtypes")
+ @Test
+ // Verify that incoming NM launched events to COMPLETED containers are
+ // handled.
+ public void testContainerCompletedAtLaunching() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+
+
+ wc.assignTaskAttempt(wc.taskAttemptID);
+
+ wc.containerCompleted();
+ wc.verifyState(AMContainerState.COMPLETED);
+ verify(wc.tal).registerRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED);
+ // TODO Failing because of an extra diagnostic event.
+
+ assertFalse(wc.amContainer.isInErrorState());
+
+ // Container launched generated by NM call.
+ wc.containerLaunched();
+ wc.verifyNoOutgoingEvents();
+
+ assertFalse(wc.amContainer.isInErrorState());
+ }
+
+ @Ignore
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testContainerCompletedAtIdle() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.containerLaunched();
+ wc.verifyState(AMContainerState.IDLE);
+
+ wc.containerCompleted();
+ wc.verifyState(AMContainerState.COMPLETED);
+ verify(wc.tal).registerRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.chh).register(wc.containerID);
+ verify(wc.chh).unregister(wc.containerID);
+
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED);
+ // TODO Failing because of two extra diagnostic event.
+
+ assertFalse(wc.amContainer.isInErrorState());
+
+ // Pending pull request. (Ideally, container should be dead at this point
+ // and this event should not be generated. Network timeout on NM-RM heartbeat
+ // can cause it to be genreated)
+ wc.pullTaskToRun();
+ wc.verifyNoOutgoingEvents();
+
+ assertFalse(wc.amContainer.isInErrorState());
+ }
+
+ @Ignore
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testContainerCompletedAtRunning() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.containerLaunched();
+ wc.pullTaskToRun();
+ wc.verifyState(AMContainerState.RUNNING);
+
+ wc.containerCompleted();
+ wc.verifyState(AMContainerState.COMPLETED);
+ verify(wc.tal).registerRunningContainer(wc.containerID);
+ verify(wc.tal).unregisterRunningContainer(wc.containerID);
+ verify(wc.chh).register(wc.containerID);
+ verify(wc.chh).unregister(wc.containerID);
+
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED);
+ // TODO Failing because of two extra diagnostic event.
+
+ assertFalse(wc.amContainer.isInErrorState());
+
+ // Pending task complete. (Ideally, container should be dead at this point
+ // and this event should not be generated. Network timeout on NM-RM heartbeat
+ // can cause it to be genreated)
+ wc.taskAttemptSucceeded(wc.taskAttemptID);
+ wc.verifyNoOutgoingEvents();
+
+ assertFalse(wc.amContainer.isInErrorState());
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testTaskAssignedToCompletedContainer() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+ wc.containerLaunched();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.pullTaskToRun();
+ wc.taskAttemptSucceeded(wc.taskAttemptID);
+
+ wc.containerCompleted();
+ wc.verifyState(AMContainerState.COMPLETED);
+
+ TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+
+ wc.assignTaskAttempt(taID2);
+
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED);
+ TaskAttemptEventContainerTerminated ctEvent =
+ (TaskAttemptEventContainerTerminated) outgoingEvents.get(0);
+ assertEquals(taID2, ctEvent.getTaskAttemptID());
+
+ // Allocation to a completed Container is considered an error.
+ // TODO Is this valid ?
+ assertTrue(wc.amContainer.isInErrorState());
+ }
+
+ @Test
+ public void testTaskPullAtLaunching() {
+ WrappedContainer wc = new WrappedContainer();
+
+ wc.launchContainer();
+ AMContainerTask pulledTask = wc.pullTaskToRun();
+ wc.verifyState(AMContainerState.LAUNCHING);
+ wc.verifyNoOutgoingEvents();
+ assertFalse(pulledTask.shouldDie());
+ assertNull(pulledTask.getTask());
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testNodeFailedAtIdle() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+ wc.containerLaunched();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.verifyState(AMContainerState.IDLE);
+
+ wc.nodeFailed();
+ // Expecting a complete event from the RM
+ wc.verifyState(AMContainerState.STOPPING);
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_NODE_FAILED,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+ AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
+
+ for (Event event : outgoingEvents) {
+ if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
+ TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
+ assertEquals("nodeFailed", nfEvent.getDiagnosticInfo());
+ }
+ }
+
+ wc.containerCompleted();
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+ assertFalse(wc.amContainer.isInErrorState());
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testNodeFailedAtIdleMultipleAttempts() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+ wc.containerLaunched();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.pullTaskToRun();
+ wc.taskAttemptSucceeded(wc.taskAttemptID);
+ wc.verifyState(AMContainerState.IDLE);
+
+ TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+ wc.assignTaskAttempt(taID2);
+ wc.pullTaskToRun();
+ wc.taskAttemptSucceeded(taID2);
+ wc.verifyState(AMContainerState.IDLE);
+
+ wc.nodeFailed();
+ // Expecting a complete event from the RM
+ wc.verifyState(AMContainerState.STOPPING);
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_NODE_FAILED,
+ TaskAttemptEventType.TA_NODE_FAILED,
+ AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
+
+ for (Event event : outgoingEvents) {
+ if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
+ TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
+ assertEquals("nodeFailed", nfEvent.getDiagnosticInfo());
+ }
+ }
+
+ assertFalse(wc.amContainer.isInErrorState());
+
+ wc.containerCompleted();
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+ assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size());
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testNodeFailedAtRunningMultipleAttempts() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+ wc.containerLaunched();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.pullTaskToRun();
+ wc.taskAttemptSucceeded(wc.taskAttemptID);
+
+ TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+ wc.assignTaskAttempt(taID2);
+ wc.pullTaskToRun();
+ wc.verifyState(AMContainerState.RUNNING);
+
+ wc.nodeFailed();
+ // Expecting a complete event from the RM
+ wc.verifyState(AMContainerState.STOPPING);
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(4);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_NODE_FAILED,
+ TaskAttemptEventType.TA_NODE_FAILED,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATING,
+ AMSchedulerEventType.S_CONTAINER_DEALLOCATE);
+
+ for (Event event : outgoingEvents) {
+ if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
+ TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
+ assertEquals("nodeFailed", nfEvent.getDiagnosticInfo());
+ }
+ }
+
+ wc.containerCompleted();
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_CONTAINER_TERMINATED,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+ assertFalse(wc.amContainer.isInErrorState());
+// assertNull(wc.amContainer.getRunningTaskAttempt()); // TODO. Set/Unset properly.
+// assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size()); // TODO. Set/Unset properly.
+// assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size()); // TODO. Set/Unset properly.
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testNodeFailedAtCompletedMultipleSuccessfulTAs() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+ wc.containerLaunched();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.pullTaskToRun();
+ wc.taskAttemptSucceeded(wc.taskAttemptID);
+
+ TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+ wc.assignTaskAttempt(taID2);
+ wc.pullTaskToRun();
+ wc.taskAttemptSucceeded(taID2);
+ wc.stopRequest();
+ wc.nmStopSent();
+ wc.containerCompleted();
+ wc.verifyState(AMContainerState.COMPLETED);
+
+ wc.nodeFailed();
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ TaskAttemptEventType.TA_NODE_FAILED,
+ TaskAttemptEventType.TA_NODE_FAILED);
+
+ assertNull(wc.amContainer.getRunningTaskAttempt());
+ assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
+ assertEquals(2, wc.amContainer.getCompletedTaskAttempts().size());
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void testDuplicateCompletedEvents() {
+ WrappedContainer wc = new WrappedContainer();
+ List<Event> outgoingEvents;
+
+ wc.launchContainer();
+ wc.containerLaunched();
+ wc.assignTaskAttempt(wc.taskAttemptID);
+ wc.pullTaskToRun();
+ wc.taskAttemptSucceeded(wc.taskAttemptID);
+
+ TezTaskAttemptID taID2 = new TezTaskAttemptID(wc.taskID, 2);
+ wc.assignTaskAttempt(taID2);
+ wc.pullTaskToRun();
+ wc.taskAttemptSucceeded(taID2);
+ wc.stopRequest();
+ wc.nmStopSent();
+ wc.containerCompleted();
+ wc.verifyState(AMContainerState.COMPLETED);
+
+ outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
+ verifyUnOrderedOutgoingEventTypes(outgoingEvents,
+ AMSchedulerEventType.S_CONTAINER_COMPLETED);
+
+ wc.containerCompleted();
+ wc.verifyNoOutgoingEvents();
+ }
+
+
+ // TODO Verify diagnostics in most of the tests.
+
+ private static class WrappedContainer {
+
+ long rmIdentifier = 2000;
+ ApplicationId applicationID;
+ ApplicationAttemptId appAttemptID;
+ ContainerId containerID;
+ NodeId nodeID;
+ String nodeHttpAddress;
+ Resource resource;
+ Priority priority;
+ Container container;
+ ContainerHeartbeatHandler chh;
+ TaskAttemptListener tal;
+
+ @SuppressWarnings("rawtypes")
+ EventHandler eventHandler;
+
+ AppContext appContext;
+
+ TezDAGID dagID;
+ TezVertexID vertexID;
+ TezTaskID taskID;
+ TezTaskAttemptID taskAttemptID;
+
+ TezTaskContext tezTaskContext;
+
+ Token<JobTokenIdentifier> jobToken;
+
+ public AMContainerImpl amContainer;
+
+ @SuppressWarnings("unchecked")
+ public WrappedContainer() {
+ applicationID = BuilderUtils.newApplicationId(rmIdentifier, 1);
+ appAttemptID = BuilderUtils.newApplicationAttemptId(applicationID, 1);
+ containerID = BuilderUtils.newContainerId(appAttemptID, 1);
+ nodeID = BuilderUtils.newNodeId("host", 12500);
+ nodeHttpAddress = "host:12501";
+ resource = BuilderUtils.newResource(1024, 1);
+ priority = BuilderUtils.newPriority(1);
+ container = BuilderUtils.newContainer(containerID, nodeID,
+ nodeHttpAddress, resource, priority, null, rmIdentifier);
+ chh = mock(ContainerHeartbeatHandler.class);
+
+ InetSocketAddress addr = new InetSocketAddress("localhost", 0);
+ tal = mock(TaskAttemptListener.class);
+ doReturn(addr).when(tal).getAddress();
+
+ eventHandler = mock(EventHandler.class);
+
+ appContext = mock(AppContext.class);
+ doReturn(new HashMap<ApplicationAccessType, String>()).when(appContext)
+ .getApplicationACLs();
+ doReturn(eventHandler).when(appContext).getEventHandler();
+
+ dagID = new TezDAGID(applicationID, 1);
+ vertexID = new TezVertexID(dagID, 1);
+ taskID = new TezTaskID(vertexID, 1);
+ taskAttemptID = new TezTaskAttemptID(taskID, 1);
+
+ tezTaskContext = mock(TezTaskContext.class);
+ doReturn(taskAttemptID).when(tezTaskContext).getTaskAttemptId();
+
+
+ jobToken = (Token<JobTokenIdentifier>) mock(Token.class);
+
+ amContainer = new AMContainerImpl(container, chh, tal,
+ appContext);
+ }
+
+ /**
+ * Verifies no additional outgoing events generated by the last incoming
+ * event to the AMContainer.
+ */
+ @SuppressWarnings("unchecked")
+ public void verifyNoOutgoingEvents() {
+ verify(eventHandler, never()).handle(any(Event.class));
+ }
+
+ /**
+ * Returns a list of outgoing events generated by the last incoming event to
+ * the AMContainer.
+ * @param invocations number of expected invocations.
+ *
+ * @return a list of outgoing events from the AMContainer.
+ */
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public List<Event> verifyCountAndGetOutgoingEvents(int invocations) {
+ ArgumentCaptor<Event> args = ArgumentCaptor.forClass(Event.class);
+ verify(eventHandler, times(invocations)).handle(args.capture());
+ return args.getAllValues();
+ }
+
+ public void launchContainer() {
+ reset(eventHandler);
+ amContainer.handle(new AMContainerEventLaunchRequest(containerID, vertexID,
+ jobToken, new Credentials(), false, new TezConfiguration(),
+ new HashMap<String, LocalResource>(), new HashMap<String, String>(),
+ null));
+ }
+
+ public void assignTaskAttempt(TezTaskAttemptID taID) {
+ reset(eventHandler);
+ amContainer.handle(new AMContainerEventAssignTA(containerID, taID,
+ tezTaskContext));
+ }
+
+ public AMContainerTask pullTaskToRun() {
+ reset(eventHandler);
+ return amContainer.pullTaskContext();
+ }
+
+ public void containerLaunched() {
+ reset(eventHandler);
+ amContainer.handle(new AMContainerEventLaunched(containerID, 3000));
+ }
+
+ public void taskAttemptSucceeded(TezTaskAttemptID taID) {
+ reset(eventHandler);
+ amContainer.handle(new AMContainerEventTASucceeded(containerID, taID));
+ }
+
+ public void stopRequest() {
+ reset(eventHandler);
+ amContainer.handle(new AMContainerEvent(containerID,
+ AMContainerEventType.C_STOP_REQUEST));
+ }
+
+ public void nmStopSent() {
+ reset(eventHandler);
+ amContainer.handle(new AMContainerEvent(containerID,
+ AMContainerEventType.C_NM_STOP_SENT));
+ }
+
+ public void nmStopFailed() {
+ reset(eventHandler);
+ amContainer.handle(new AMContainerEvent(containerID,
+ AMContainerEventType.C_NM_STOP_FAILED));
+ }
+
+ public void containerCompleted() {
+ reset(eventHandler);
+ ContainerStatus cStatus = ContainerStatus.newInstance(containerID,
+ ContainerState.COMPLETE, "", 100);
+ amContainer.handle(new AMContainerEventCompleted(cStatus));
+ }
+
+ public void containerTimedOut() {
+ reset(eventHandler);
+ amContainer.handle(new AMContainerEvent(containerID,
+ AMContainerEventType.C_TIMED_OUT));
+ }
+
+ public void launchFailed() {
+ reset(eventHandler);
+ amContainer.handle(new AMContainerEventLaunchFailed(containerID,
+ "launchFailed"));
+ }
+
+ public void nodeFailed() {
+ reset(eventHandler);
+ amContainer.handle(new AMContainerEventNodeFailed(containerID,
+ "nodeFailed"));
+ }
+
+ public void verifyState(AMContainerState state) {
+ assertEquals(
+ "Expected state: " + state + ", but found: " + amContainer.getState(),
+ state, amContainer.getState());
+ }
+ }
+
+ @SuppressWarnings("rawtypes")
+ private void verifyUnOrderedOutgoingEventTypes(List<Event> events,
+ Enum<?>... expectedTypes) {
+
+ List<Enum<?>> expectedTypeList = new LinkedList<Enum<?>>();
+ for (Enum<?> expectedType : expectedTypes) {
+ expectedTypeList.add(expectedType);
+ }
+ List<Event> eventsCopy = new LinkedList<Event>(events);
+
+ Iterator<Enum<?>> expectedTypeIterator = expectedTypeList.iterator();
+ while (expectedTypeIterator.hasNext()) {
+ Enum<?> expectedType = expectedTypeIterator.next();
+ Iterator<Event> iter = eventsCopy.iterator();
+ while (iter.hasNext()) {
+ Event e = iter.next();
+ if (e.getType() == expectedType) {
+ iter.remove();
+ expectedTypeIterator.remove();
+ break;
+ }
+ }
+ }
+ assertTrue("Did not find types : " + expectedTypeList
+ + " in outgoing event list", expectedTypeList.isEmpty());
+ assertTrue("Found unexpected events: " + eventsCopy
+ + " in outgoing event list", eventsCopy.isEmpty());
+ }
+}
[14/50] [abbrv] TEZ-9. Add support for client api and records.
cleaned up packages (bikas)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
deleted file mode 100644
index a5a9e73..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.records;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-
-public interface TaskAttemptReport {
- public abstract TezTaskAttemptID getTaskAttemptId();
- public abstract TaskAttemptState getTaskAttemptState();
- public abstract float getProgress();
- public abstract long getStartTime();
- public abstract long getFinishTime();
- /** @return the shuffle finish time. Applicable only for reduce attempts */
- public abstract long getShuffleFinishTime();
- /** @return the sort/merge finish time. Applicable only for reduce attempts */
- public abstract long getSortFinishTime();
- public abstract TezCounters getCounters();
- public abstract String getDiagnosticInfo();
- public abstract String getStateString();
- public abstract String getNodeManagerHost();
- public abstract int getNodeManagerPort();
- public abstract int getNodeManagerHttpPort();
- public abstract ContainerId getContainerId();
-
- public abstract void setTaskAttemptId(TezTaskAttemptID taskAttemptId);
- public abstract void setTaskAttemptState(TaskAttemptState taskAttemptState);
- public abstract void setProgress(float progress);
- public abstract void setStartTime(long startTime);
- public abstract void setFinishTime(long finishTime);
- public abstract void setCounters(TezCounters counters);
- public abstract void setDiagnosticInfo(String diagnosticInfo);
- public abstract void setStateString(String stateString);
- public abstract void setNodeManagerHost(String nmHost);
- public abstract void setNodeManagerPort(int nmPort);
- public abstract void setNodeManagerHttpPort(int nmHttpPort);
- public abstract void setContainerId(ContainerId containerId);
-
- /**
- * Set the shuffle finish time. Applicable only for reduce attempts
- * @param time the time the shuffle finished.
- */
- public abstract void setShuffleFinishTime(long time);
- /**
- * Set the sort/merge finish time. Applicable only for reduce attempts
- * @param time the time the shuffle finished.
- */
- public abstract void setSortFinishTime(long time);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptState.java
deleted file mode 100644
index 455aa1b..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptState.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.records;
-
-public enum TaskAttemptState {
- NEW,
- STARTING,
- RUNNING,
- COMMIT_PENDING,
- SUCCEEDED,
- FAILED,
- KILLED
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
deleted file mode 100644
index a0834d7..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.records;
-
-import java.util.List;
-
-import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-
-public interface TaskReport {
- public abstract TezTaskID getTaskId();
- public abstract TaskState getTaskState();
- public abstract float getProgress();
- public abstract long getStartTime();
- public abstract long getFinishTime();
- public abstract TezCounters getCounters();
-
- public abstract List<TezTaskAttemptID> getRunningAttemptsList();
- public abstract TezTaskAttemptID getRunningAttempt(int index);
- public abstract int getRunningAttemptsCount();
-
- public abstract TezTaskAttemptID getSuccessfulAttempt();
-
- public abstract List<String> getDiagnosticsList();
- public abstract String getDiagnostics(int index);
- public abstract int getDiagnosticsCount();
-
-
- public abstract void setTaskId(TezTaskID taskId);
- public abstract void setTaskState(TaskState taskState);
- public abstract void setProgress(float progress);
- public abstract void setStartTime(long startTime);
- public abstract void setFinishTime(long finishTime);
- public abstract void setCounters(TezCounters counters);
-
- public abstract void addAllRunningAttempts(List<TezTaskAttemptID> taskAttempts);
- public abstract void addRunningAttempt(TezTaskAttemptID taskAttempt);
- public abstract void removeRunningAttempt(int index);
- public abstract void clearRunningAttempts();
-
- public abstract void setSuccessfulAttempt(TezTaskAttemptID taskAttempt);
- public abstract void addAllDiagnostics(List<String> diagnostics);
- public abstract void addDiagnostics(String diagnostics);
- public abstract void removeDiagnostics(int index);
- public abstract void clearDiagnostics();
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskState.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskState.java
deleted file mode 100644
index 411a8f4..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskState.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.records;
-
-public enum TaskState {
- NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILLED
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index a0c2676..804c20c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -63,10 +64,14 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.VertexStatus;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.app.client.ClientService;
import org.apache.tez.dag.app.client.impl.TezClientService;
import org.apache.tez.dag.app.dag.DAG;
@@ -173,6 +178,7 @@ public class DAGAppMaster extends CompositeService {
private HistoryEventHandler historyEventHandler;
private DAGAppMasterState state;
+ private DAGMonitorServer monitor;
private DAG dag;
private Credentials fsTokens = new Credentials(); // Filled during init
@@ -223,6 +229,8 @@ public class DAGAppMaster extends CompositeService {
appName = jobPlan.getName();
dagId = new TezDAGID(appAttemptID.getApplicationId(), 1);
+
+ monitor = new DAGMonitorServer();
// TODO Committer.
// committer = createOutputCommitter(conf);
@@ -703,6 +711,35 @@ public class DAGAppMaster extends CompositeService {
}
LOG.info("On DAG completion. Old state: " + oldState + " new state: " + state);
}
+
+ class DAGMonitorServer implements DAGClient {
+
+ @Override
+ public List<String> getAllDAGs() {
+ return Collections.singletonList(dag.getID().toString());
+ }
+
+ @Override
+ public DAGStatus getDAGStatus(String dagIdStr) throws TezException {
+ return getDAG(dagIdStr).getDAGStatus();
+ }
+
+ @Override
+ public VertexStatus getVertexStatus(String dagIdStr, String vertexName) {
+ return getDAG(dagIdStr).getVertexStatus(vertexName);
+ }
+
+ DAG getDAG(String dagIdStr) {
+ TezDAGID dagId = TezDAGID.fromString(dagIdStr);
+ if(dagId == null) {
+ throw new TezException("Bad dagId: " + dagIdStr);
+ }
+ if(!dagId.equals(dag.getID())) {
+ throw new TezException("Unknown dagId: " + dagIdStr);
+ }
+ return dag;
+ }
+ }
private class RunningAppContext implements AppContext {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
index 0d845b2..92f0e87 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
@@ -17,7 +17,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index f2c3791..913b565 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -24,9 +24,10 @@ import java.util.Map;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.client.DAGStatusBuilder;
+import org.apache.tez.dag.api.client.VertexStatusBuilder;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezVertexID;
@@ -61,6 +62,8 @@ public interface DAG {
TezConfiguration getConf();
DAGPlan getJobPlan();
+ DAGStatusBuilder getDAGStatus();
+ VertexStatusBuilder getVertexStatus(String vertexName);
/**
* @return the ACLs for this job for each type of JobACL given.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java
index 1fe18fa..49ab24a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.app.dag;
import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.tez.dag.api.records.AMInfo;
+import org.apache.tez.dag.api.oldrecords.AMInfo;
public abstract class DAGReport {
public abstract ApplicationId getDAGId();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
index 6cfc2c2..f3ca831 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
@@ -21,8 +21,8 @@ package org.apache.tez.dag.app.dag;
import java.util.Map;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.records.TaskReport;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.oldrecords.TaskReport;
+import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
index d4e038a..712f968 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
@@ -25,8 +25,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.records.TaskAttemptReport;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptReport;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.records.TezTaskAttemptID;
/**
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index ace8a8b..fc86753 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -28,7 +28,9 @@ import org.apache.tez.common.InputSpec;
import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.EdgeProperty;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.client.ProgressBuilder;
+import org.apache.tez.dag.api.client.VertexStatusBuilder;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
@@ -59,6 +61,8 @@ public interface Vertex extends Comparable<Vertex> {
int getTotalTasks();
int getCompletedTasks();
float getProgress();
+ ProgressBuilder getVertexProgress();
+ VertexStatusBuilder getVertexStatus();
TezDependentTaskCompletionEvent[]
getTaskAttemptCompletionEvents(int fromEventId, int maxEvents);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
index ef23638..2659bba 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.app.dag.event;
import java.util.List;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptEventStatusUpdate extends TaskAttemptEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
index 79512c7..4bf4e24 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.app.dag.event;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.records.TezTaskID;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 02e91b4..66f9c36 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -52,14 +52,16 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGProtos.EdgePlan;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint;
-import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
-import org.apache.tez.dag.api.impl.DAGStatus;
+import org.apache.tez.dag.api.client.DAGStatusBuilder;
+import org.apache.tez.dag.api.client.ProgressBuilder;
+import org.apache.tez.dag.api.client.VertexStatusBuilder;
+import org.apache.tez.dag.api.committer.DAGStatus;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
@@ -84,6 +86,7 @@ import org.apache.tez.dag.history.events.DAGStartedEvent;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.dag.utils.DAGApps;
+import org.apache.tez.dag.utils.TezBuilderUtils;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
import org.apache.tez.engine.common.security.JobTokenSecretManager;
import org.apache.tez.engine.common.security.TokenCache;
@@ -505,7 +508,52 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
readLock.unlock();
}
}
+
+ // monitoring apis
+ @Override
+ public DAGStatusBuilder getDAGStatus() {
+ DAGStatusBuilder status = new DAGStatusBuilder();
+ int totalTaskCount = 0;
+ int totalSucceededTaskCount = 0;
+ int totalRunningTaskCount = 0;
+ int totalFailedTaskCount = 0;
+ int totalKilledTaskCount = 0;
+ readLock.lock();
+ try {
+ for(Map.Entry<String, Vertex> entry : vertexMap.entrySet()) {
+ ProgressBuilder progress = entry.getValue().getVertexProgress();
+ status.addVertexProgress(entry.getKey(), progress);
+ totalTaskCount += progress.getTotalTaskCount();
+ totalSucceededTaskCount += progress.getSucceededTaskCount();
+ totalRunningTaskCount += progress.getRunningTaskCount();
+ totalFailedTaskCount += progress.getFailedTaskCount();
+ totalKilledTaskCount += progress.getKilledTaskCount();
+ }
+ ProgressBuilder dagProgress = new ProgressBuilder();
+ dagProgress.setTotalTaskCount(totalTaskCount);
+ dagProgress.setSucceededTaskCount(totalSucceededTaskCount);
+ dagProgress.setRunningTaskCount(totalRunningTaskCount);
+ dagProgress.setFailedTaskCount(totalFailedTaskCount);
+ dagProgress.setKilledTaskCount(totalKilledTaskCount);
+ status.setState(getState());
+ status.setDiagnostics(diagnostics);
+ status.setDAGProgress(dagProgress);
+ return status;
+ } finally {
+ readLock.unlock();
+ }
+ }
+ @Override
+ public VertexStatusBuilder getVertexStatus(String vertexName) {
+ Vertex vertex = vertexMap.get(vertexName);
+ if(vertex == null) {
+ return null;
+ }
+ return vertex.getVertexStatus();
+ }
+
+
protected void startRootVertices() {
for (Vertex v : vertices.values()) {
if (v.getInputVerticesCount() == 0) {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 264b3ad..92ccffe 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -56,9 +56,8 @@ import org.apache.tez.common.counters.DAGCounter;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
-import org.apache.tez.dag.api.records.TaskAttemptReport;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptReport;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
@@ -93,6 +92,7 @@ import org.apache.tez.dag.history.events.TaskAttemptFinishedEvent;
import org.apache.tez.dag.history.events.TaskAttemptStartedEvent;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.utils.TezBuilderUtils;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 4d0c4f6..401f1b0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -43,9 +43,9 @@ import org.apache.hadoop.yarn.util.Records;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.records.TaskAttemptState;
-import org.apache.tez.dag.api.records.TaskReport;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskReport;
+import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index aedb386..20d13cd 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -55,19 +55,21 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.tez.common.InputSpec;
import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.oldrecords.TaskState;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.impl.NullVertexOutputCommitter;
-import org.apache.tez.dag.api.impl.VertexContext;
-import org.apache.tez.dag.api.impl.VertexOutputCommitter;
-import org.apache.tez.dag.api.impl.VertexStatus;
-import org.apache.tez.dag.api.impl.VertexStatus.State;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.client.ProgressBuilder;
+import org.apache.tez.dag.api.client.VertexStatusBuilder;
+import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
+import org.apache.tez.dag.api.committer.VertexContext;
+import org.apache.tez.dag.api.committer.VertexOutputCommitter;
+import org.apache.tez.dag.api.committer.VertexStatus;
+import org.apache.tez.dag.api.committer.VertexStatus.State;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
@@ -519,6 +521,36 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
this.readLock.unlock();
}
}
+
+ @Override
+ public ProgressBuilder getVertexProgress() {
+ this.readLock.lock();
+ try {
+ ProgressBuilder progress = new ProgressBuilder();
+ progress.setTotalTaskCount(numTasks);
+ progress.setSucceededTaskCount(succeededTaskCount);
+ progress.setRunningTaskCount(0); // TODO TEZ-130
+ progress.setFailedTaskCount(failedTaskCount);
+ progress.setKilledTaskCount(killedTaskCount);
+ return progress;
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
+ public VertexStatusBuilder getVertexStatus() {
+ this.readLock.lock();
+ try {
+ VertexStatusBuilder status = new VertexStatusBuilder();
+ status.setState(getInternalState());
+ status.setDiagnostics(diagnostics);
+ status.setProgress(getVertexProgress());
+ return status;
+ } finally {
+ this.readLock.unlock();
+ }
+ }
private void computeProgress() {
this.readLock.lock();
@@ -599,9 +631,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
writeLock.unlock();
}
}
-
- @Private
- public VertexState getInternalState() {
+
+ private VertexState getInternalState() {
readLock.lock();
try {
return getStateMachine().getCurrentState();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
index fddd204..b2a90f2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.app.rm;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.records.TezTaskAttemptID;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index 31005b6..85ba0b2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.service.AbstractService;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.DAGAppMaster;
import org.apache.tez.dag.app.DAGAppMasterState;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
index 04c104a..79ccb8e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/DefaultSpeculator.java
@@ -38,8 +38,7 @@ import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.service.AbstractService;
-import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.Task;
@@ -50,6 +49,7 @@ import org.apache.tez.dag.app.dag.event.TaskEventType;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.dag.utils.TezBuilderUtils;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
// FIXME does not handle multiple vertices
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
index aa95f37..34bb4e3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/LegacyTaskRuntimeEstimator.java
@@ -23,7 +23,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttempt;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
index 191d0b9..ba5bfda 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/speculate/StartEndTimesBase.java
@@ -26,7 +26,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.Task;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
index d1b87b1..5dea9a7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.history.events;
-import org.apache.tez.dag.api.impl.DAGStatus;
+import org.apache.tez.dag.api.committer.DAGStatus;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.DAGFinished;
import org.apache.tez.dag.history.avro.HistoryEventType;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
index 4a2917e..395ba93 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.history.events;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
import org.apache.tez.dag.history.avro.TaskAttemptFinished;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
index 165dc20..60f810b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.history.events;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
import org.apache.tez.dag.history.avro.TaskFinished;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
index b02be1e..91c209f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.history.events;
-import org.apache.tez.dag.api.impl.VertexStatus;
+import org.apache.tez.dag.api.committer.VertexStatus;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
import org.apache.tez.dag.history.avro.VertexFinished;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
new file mode 100644
index 0000000..60ec0a0
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.utils;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.dag.api.oldrecords.AMInfo;
+import org.apache.tez.dag.app.dag.DAGReport;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+
+public class TezBuilderUtils {
+
+ public static TezVertexID newVertexID(TezDAGID dagId, int vertexId) {
+ return new TezVertexID(dagId, vertexId);
+ }
+
+ public static TezTaskAttemptID newTaskAttemptId(TezTaskID taskId, int id) {
+ return new TezTaskAttemptID(taskId, id);
+ }
+
+ public static DAGReport newDAGReport() {
+ return null;
+ }
+
+ public static AMInfo newAMInfo(ApplicationAttemptId appAttemptID,
+ long startTime, ContainerId containerID, String nmHost,
+ int nmPort, int nmHttpPort) {
+ return null;
+ }
+
+ public static TezTaskID newTaskId(TezDAGID dagId, int vertexId, int taskId) {
+ return new TezTaskID(newVertexID(dagId, vertexId), taskId);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index 369bced..9601418 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -56,7 +56,7 @@ import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.common.TezTaskContext;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.records.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.ContainerHeartbeatHandler;
import org.apache.tez.dag.app.TaskAttemptListener;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 100f2aa..44e7686 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -43,8 +43,8 @@ import org.apache.hadoop.yarn.event.InlineDispatcher;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.tez.dag.api.records.TaskAttemptState;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
+import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index a1e450c..fe6ae14 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -37,23 +37,23 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.DAGProtos.EdgePlan;
-import org.apache.tez.dag.api.DAGProtos.PlanEdgeConnectionPattern;
-import org.apache.tez.dag.api.DAGProtos.PlanEdgeSourceType;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
-import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
-import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
-import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.oldrecords.TaskState;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeConnectionPattern;
+import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeSourceType;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint;
-import org.apache.tez.dag.api.impl.NullVertexOutputCommitter;
-import org.apache.tez.dag.api.impl.VertexContext;
-import org.apache.tez.dag.api.impl.VertexOutputCommitter;
-import org.apache.tez.dag.api.impl.VertexStatus.State;
-import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
+import org.apache.tez.dag.api.committer.VertexContext;
+import org.apache.tez.dag.api.committer.VertexOutputCommitter;
+import org.apache.tez.dag.api.committer.VertexStatus.State;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/de5ee038/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index c0c9923..fcb2ca1 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -99,7 +99,7 @@ import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
[37/50] [abbrv] git commit: Include container log urls and counters
as part of history events in the AM logs.
Posted by ss...@apache.org.
Include container log urls and counters as part of history events in the AM logs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/6d5625f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/6d5625f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/6d5625f4
Branch: refs/heads/master
Commit: 6d5625f4fa38cf35e6f7201fb4b18c47ec6964d3
Parents: 34de34b
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu May 30 13:34:38 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu May 30 13:34:38 2013 -0700
----------------------------------------------------------------------
tez-dag/src/main/avro/HistoryEvents.avpr | 35 +++++++-
.../org/apache/tez/dag/app/dag/impl/DAGImpl.java | 5 +-
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 27 ++++++-
.../org/apache/tez/dag/app/dag/impl/TaskImpl.java | 5 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 6 +-
.../apache/tez/dag/history/events/AvroUtils.java | 53 ++++++++++++
.../tez/dag/history/events/DAGFinishedEvent.java | 9 ++-
.../history/events/TaskAttemptFinishedEvent.java | 10 ++-
.../history/events/TaskAttemptStartedEvent.java | 11 ++-
.../tez/dag/history/events/TaskFinishedEvent.java | 9 ++-
.../dag/history/events/VertexFinishedEvent.java | 10 ++-
.../tez/dag/app/rm/container/TestAMContainer.java | 3 +-
.../apache/tez/mapreduce/examples/MRRSleepJob.java | 65 +++++++++------
13 files changed, 200 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/avro/HistoryEvents.avpr
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/avro/HistoryEvents.avpr b/tez-dag/src/main/avro/HistoryEvents.avpr
index 04333e6..8805c6d 100644
--- a/tez-dag/src/main/avro/HistoryEvents.avpr
+++ b/tez-dag/src/main/avro/HistoryEvents.avpr
@@ -21,6 +21,29 @@
"types": [
+ {"type": "record", "name": "TezCounter",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "displayName", "type": "string"},
+ {"name": "value", "type": "long"}
+ ]
+ },
+
+ {"type": "record", "name": "TezCounterGroup",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "displayName", "type": "string"},
+ {"name": "counts", "type": {"type": "array", "items": "TezCounter"}}
+ ]
+ },
+
+ {"type": "record", "name": "TezCounters",
+ "fields": [
+ {"name": "groups", "type": {"type": "array", "items": "TezCounterGroup"}}
+ ]
+ },
+
+
{"type": "record", "name": "AMStarted",
"fields": [
{"name": "applicationAttemptId", "type": "string"},
@@ -50,7 +73,8 @@
{"name": "dagId", "type": "string"},
{"name": "finishTime", "type": "long"},
{"name": "status", "type": "string"},
- {"name": "diagnostics", "type": "string"}
+ {"name": "diagnostics", "type": "string"},
+ {"name": "counters", "type": "TezCounters"}
]
},
@@ -71,7 +95,8 @@
{"name": "vertexId", "type": "string"},
{"name": "finishTime", "type": "long"},
{"name": "status", "type": "string"},
- {"name": "diagnostics", "type": "string"}
+ {"name": "diagnostics", "type": "string"},
+ {"name": "counters", "type": "TezCounters"}
]
},
@@ -89,7 +114,8 @@
{"name": "vertexName", "type": "string"},
{"name": "taskId", "type": "string"},
{"name": "finishTime", "type": "long"},
- {"name": "status", "type": "string"}
+ {"name": "status", "type": "string"},
+ {"name": "counters", "type": "TezCounters"}
]
},
@@ -109,7 +135,8 @@
{"name": "taskAttemptId", "type": "string"},
{"name": "finishTime", "type": "long"},
{"name": "status", "type": "string"},
- {"name": "diagnostics", "type": "string"}
+ {"name": "diagnostics", "type": "string"},
+ {"name": "counters", "type": "TezCounters"}
]
},
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 3681c85..d2395ad 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -625,7 +625,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
void logJobHistoryFinishedEvent() {
this.setFinishTime();
DAGFinishedEvent finishEvt = new DAGFinishedEvent(dagId, finishTime,
- DAGStatus.State.SUCCEEDED, "");
+ DAGStatus.State.SUCCEEDED, "", getAllCounters());
this.eventHandler.handle(
new DAGHistoryEvent(dagId, finishEvt));
}
@@ -641,7 +641,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
void logJobHistoryUnsuccesfulEvent(DAGStatus.State state) {
DAGFinishedEvent finishEvt = new DAGFinishedEvent(dagId, clock.getTime(),
- state, StringUtils.join(LINE_SEPARATOR, getDiagnostics()));
+ state, StringUtils.join(LINE_SEPARATOR, getDiagnostics()),
+ getAllCounters());
this.eventHandler.handle(
new DAGHistoryEvent(dagId, finishEvt));
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 5225ecf..c14ba8a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
@@ -794,9 +795,27 @@ public class TaskAttemptImpl implements TaskAttempt,
@SuppressWarnings("unchecked")
protected void logJobHistoryAttemptStarted() {
+ final String containerIdStr = containerId.toString();
+ String inProgressLogsUrl = nodeHttpAddress
+ + "/" + "node/containerlogs"
+ + "/" + containerIdStr
+ + "/" + this.appContext.getUser();
+ String completedLogsUrl = "";
+ if (conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
+ YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)
+ && conf.get(YarnConfiguration.YARN_LOG_SERVER_URL) != null) {
+ String contextStr = "v_" + getTask().getVertex().getName()
+ + "_" + this.attemptId.toString();
+ completedLogsUrl = conf.get(YarnConfiguration.YARN_LOG_SERVER_URL)
+ + "/" + containerNodeId.toString()
+ + "/" + containerIdStr
+ + "/" + contextStr
+ + "/" + this.appContext.getUser();
+ }
TaskAttemptStartedEvent startEvt = new TaskAttemptStartedEvent(
attemptId, getTask().getVertex().getName(),
- launchTime, containerId, containerNodeId);
+ launchTime, containerId, containerNodeId,
+ inProgressLogsUrl, completedLogsUrl);
eventHandler.handle(new DAGHistoryEvent(
attemptId.getTaskID().getVertexID().getDAGId(),
startEvt));
@@ -809,7 +828,8 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptFinishedEvent finishEvt = new TaskAttemptFinishedEvent(
attemptId, getTask().getVertex().getName(),
- getFinishTime(), TaskAttemptState.SUCCEEDED, "");
+ getFinishTime(), TaskAttemptState.SUCCEEDED, "",
+ getCounters());
// FIXME how do we store information regd completion events
eventHandler.handle(new DAGHistoryEvent(
attemptId.getTaskID().getVertexID().getDAGId(),
@@ -823,7 +843,8 @@ public class TaskAttemptImpl implements TaskAttempt,
attemptId, getTask().getVertex().getName(),
clock.getTime(), state,
StringUtils.join(
- LINE_SEPARATOR, getDiagnostics()));
+ LINE_SEPARATOR, getDiagnostics()),
+ getCounters());
// FIXME how do we store information regd completion events
eventHandler.handle(new DAGHistoryEvent(
attemptId.getTaskID().getVertexID().getDAGId(),
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index fb738ce..575f32c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -806,14 +806,15 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
// FIXME need to handle getting finish time as this function
// is called from within a transition
TaskFinishedEvent finishEvt = new TaskFinishedEvent(taskId,
- getVertex().getName(), clock.getTime(), TaskState.SUCCEEDED);
+ getVertex().getName(), clock.getTime(), TaskState.SUCCEEDED,
+ getCounters());
this.eventHandler.handle(new DAGHistoryEvent(
taskId.getVertexID().getDAGId(), finishEvt));
}
protected void logJobHistoryTaskFailedEvent(TaskState finalState) {
TaskFinishedEvent finishEvt = new TaskFinishedEvent(taskId,
- getVertex().getName(), clock.getTime(), finalState);
+ getVertex().getName(), clock.getTime(), finalState, getCounters());
this.eventHandler.handle(new DAGHistoryEvent(
taskId.getVertexID().getDAGId(), finishEvt));
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index e34382f..c7bceb6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -670,14 +670,16 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
void logJobHistoryVertexFinishedEvent() {
this.setFinishTime();
VertexFinishedEvent finishEvt = new VertexFinishedEvent(vertexId,
- vertexName, finishTime, VertexStatus.State.SUCCEEDED, "");
+ vertexName, finishTime, VertexStatus.State.SUCCEEDED, "",
+ getAllCounters());
this.eventHandler.handle(new DAGHistoryEvent(getDAGId(), finishEvt));
}
void logJobHistoryVertexFailedEvent(VertexStatus.State state) {
VertexFinishedEvent finishEvt = new VertexFinishedEvent(vertexId,
vertexName, clock.getTime(), state,
- StringUtils.join(LINE_SEPARATOR, getDiagnostics()));
+ StringUtils.join(LINE_SEPARATOR, getDiagnostics()),
+ getAllCounters());
this.eventHandler.handle(new DAGHistoryEvent(getDAGId(), finishEvt));
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/AvroUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AvroUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AvroUtils.java
new file mode 100644
index 0000000..3587964
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AvroUtils.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.history.events;
+
+import java.util.ArrayList;
+
+import org.apache.avro.util.Utf8;
+import org.apache.tez.common.counters.CounterGroup;
+import org.apache.tez.dag.history.avro.TezCounter;
+import org.apache.tez.dag.history.avro.TezCounterGroup;
+import org.apache.tez.dag.history.avro.TezCounters;
+
+public class AvroUtils {
+
+ public static TezCounters toAvro(
+ org.apache.tez.common.counters.TezCounters counters) {
+ TezCounters result = new TezCounters();
+ result.groups = new ArrayList<TezCounterGroup>(0);
+ if (counters == null) return result;
+ for (CounterGroup group : counters) {
+ TezCounterGroup g = new TezCounterGroup();
+ g.name = new Utf8(group.getName());
+ g.displayName = new Utf8(group.getDisplayName());
+ g.counts = new ArrayList<TezCounter>(group.size());
+ for (org.apache.tez.common.counters.TezCounter counter : group) {
+ TezCounter c = new TezCounter();
+ c.name = new Utf8(counter.getName());
+ c.displayName = new Utf8(counter.getDisplayName());
+ c.value = counter.getValue();
+ g.counts.add(c);
+ }
+ result.groups.add(g);
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
index 8794cea..8ded3d4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
@@ -18,6 +18,7 @@
package org.apache.tez.dag.history.events;
+import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.DAGFinished;
@@ -27,14 +28,17 @@ import org.apache.tez.dag.records.TezDAGID;
public class DAGFinishedEvent implements HistoryEvent {
private DAGFinished datum = new DAGFinished();
+ // FIXME remove this when we have a proper history
+ private final TezCounters tezCounters;
public DAGFinishedEvent(TezDAGID dagId,
long finishTime, DAGStatus.State state,
- String diagnostics) {
+ String diagnostics, TezCounters counters) {
datum.dagId = dagId.toString();
datum.finishTime = finishTime;
datum.status = state.name();
datum.diagnostics = diagnostics;
+ tezCounters = counters;
}
@Override
@@ -58,6 +62,7 @@ public class DAGFinishedEvent implements HistoryEvent {
return "dagId=" + datum.dagId
+ ", finishTime=" + datum.finishTime
+ ", status=" + datum.status
- + ", diagnostics=" + datum.diagnostics;
+ + ", diagnostics=" + datum.diagnostics
+ + ", counters=" + tezCounters.toString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
index 395ba93..9794938 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java
@@ -18,6 +18,7 @@
package org.apache.tez.dag.history.events;
+import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
@@ -27,17 +28,21 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptFinishedEvent implements HistoryEvent {
private TaskAttemptFinished datum = new TaskAttemptFinished();
+ // FIXME remove this when we have a proper history
+ private final TezCounters tezCounters;
public TaskAttemptFinishedEvent(TezTaskAttemptID taId,
String vertexName,
long finishTime,
TaskAttemptState state,
- String diagnostics) {
+ String diagnostics,
+ TezCounters counters) {
datum.taskAttemptId = taId.toString();
datum.vertexName = vertexName;
datum.finishTime = finishTime;
datum.status = state.name();
datum.diagnostics = diagnostics;
+ tezCounters = counters;
}
@Override
@@ -62,6 +67,7 @@ public class TaskAttemptFinishedEvent implements HistoryEvent {
+ ", taskAttemptId=" + datum.taskAttemptId
+ ", finishTime=" + datum.finishTime
+ ", status=" + datum.status
- + ", diagnostics=" + datum.diagnostics;
+ + ", diagnostics=" + datum.diagnostics
+ + ", counters=" + tezCounters.toString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
index 97c9f2f..5d29b21 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java
@@ -28,15 +28,20 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptStartedEvent implements HistoryEvent {
private TaskAttemptStarted datum = new TaskAttemptStarted();
+ private final String inProgressLogsUrl;
+ private final String completedLogsUrl;
public TaskAttemptStartedEvent(TezTaskAttemptID taId,
String vertexName, long startTime,
- ContainerId containerId, NodeId nodeId) {
+ ContainerId containerId, NodeId nodeId,
+ String inProgressLogsUrl, String completedLogsUrl) {
datum.taskAttemptId = taId.toString();
datum.vertexName = vertexName;
datum.startTime = startTime;
datum.containerId = containerId.toString();
datum.nodeId = nodeId.toString();
+ this.inProgressLogsUrl = inProgressLogsUrl;
+ this.completedLogsUrl = completedLogsUrl;
}
@Override
@@ -61,6 +66,8 @@ public class TaskAttemptStartedEvent implements HistoryEvent {
+ ", taskAttemptId=" + datum.taskAttemptId
+ ", startTime=" + datum.startTime
+ ", containerId=" + datum.containerId
- + ", nodeId=" + datum.nodeId;
+ + ", nodeId=" + datum.nodeId
+ + ", inProgressLogs=" + inProgressLogsUrl
+ + ", completedLogs=" + completedLogsUrl;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
index 60f810b..5408a72 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
@@ -18,6 +18,7 @@
package org.apache.tez.dag.history.events;
+import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
@@ -27,14 +28,17 @@ import org.apache.tez.dag.records.TezTaskID;
public class TaskFinishedEvent implements HistoryEvent {
private TaskFinished datum = new TaskFinished();
+ // FIXME remove this when we have a proper history
+ private final TezCounters tezCounters;
public TaskFinishedEvent(TezTaskID taskId,
String vertexName, long finishTime,
- TaskState state) {
+ TaskState state, TezCounters counters) {
datum.vertexName = vertexName;
datum.taskId = taskId.toString();
datum.finishTime = finishTime;
datum.status = state.name();
+ tezCounters = counters;
}
@Override
@@ -58,6 +62,7 @@ public class TaskFinishedEvent implements HistoryEvent {
return "vertexName=" + datum.vertexName
+ ", taskId=" + datum.taskId
+ ", finishTime=" + datum.finishTime
- + ", status=" + datum.status;
+ + ", status=" + datum.status
+ + ", counters=" + tezCounters.toString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
index 7e86c66..ec13ebd 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
@@ -18,6 +18,7 @@
package org.apache.tez.dag.history.events;
+import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.client.VertexStatus;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
@@ -27,15 +28,19 @@ import org.apache.tez.dag.records.TezVertexID;
public class VertexFinishedEvent implements HistoryEvent {
private VertexFinished datum = new VertexFinished();
+ // FIXME remove this when we have a proper history
+ private final TezCounters tezCounters;
public VertexFinishedEvent(TezVertexID vertexId,
String vertexName, long finishTime,
- VertexStatus.State state, String diagnostics) {
+ VertexStatus.State state, String diagnostics,
+ TezCounters counters) {
datum.vertexName = vertexName;
datum.vertexId = vertexId.toString();
datum.finishTime = finishTime;
datum.status = state.name();
datum.diagnostics = diagnostics;
+ tezCounters = counters;
}
@Override
@@ -60,6 +65,7 @@ public class VertexFinishedEvent implements HistoryEvent {
+ ", vertexId=" + datum.vertexId
+ ", finishTime=" + datum.finishTime
+ ", status=" + datum.status
- + ", diagnostics=" + datum.diagnostics;
+ + ", diagnostics=" + datum.diagnostics
+ + ", counters=" + tezCounters.toString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index 4541a2e..cf01f43 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -839,7 +839,8 @@ public class TestAMContainer {
resource = BuilderUtils.newResource(1024, 1);
priority = BuilderUtils.newPriority(1);
container = BuilderUtils.newContainer(containerID, nodeID,
- nodeHttpAddress, resource, priority, null, rmIdentifier);
+ nodeHttpAddress, resource, priority, null);
+
chh = mock(ContainerHeartbeatHandler.class);
InetSocketAddress addr = new InetSocketAddress("localhost", 0);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6d5625f4/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
index 6115c6f..627d583 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -72,6 +72,8 @@ public class MRRSleepJob extends Configured implements Tool {
"mrr.sleepjob.ireduce.sleep.time";
public static String IREDUCE_STAGES_COUNT =
"mrr.sleepjob.ireduces.stages.count";
+ public static String IREDUCE_TASKS_COUNT =
+ "mrr.sleepjob.ireduces.tasks.count";
// Flags to inject failures
public static String MAP_THROW_ERROR = "mrr.sleepjob.map.throw.error";
@@ -109,28 +111,21 @@ public class MRRSleepJob extends Configured implements Tool {
InputSplit ignored, TaskAttemptContext taskContext)
throws IOException {
Configuration conf = taskContext.getConfiguration();
- String vertexName = conf.get(
- org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
- boolean isIntermediateReduce =
- MultiStageMRConfigUtil.getIntermediateStageNum(vertexName) != -1;
-
- final int count =
- (isIntermediateReduce)?
- conf.getInt(IREDUCE_SLEEP_COUNT, 1) :
- conf.getInt(MAP_SLEEP_COUNT, 1);
- if (count < 0) throw new IOException("Invalid map count: " + count);
- int totalIReduces = MultiStageMRConfigUtil.getNumIntermediateStages(conf);
- boolean finalIReduce = totalIReduces ==
- (MultiStageMRConfigUtil.getIntermediateStageNum(vertexName) + 1);
+ final int count = conf.getInt(MAP_SLEEP_COUNT, 1);
+ if (count < 0) {
+ throw new IOException("Invalid map count: " + count);
+ }
- final int redcount = finalIReduce?
- conf.getInt(REDUCE_SLEEP_COUNT, 1) :
- conf.getInt(IREDUCE_SLEEP_COUNT, 1);
- if (redcount < 0)
- throw new IOException("Invalid reduce count: " + redcount);
+ int totalIReduces = conf.getInt(IREDUCE_STAGES_COUNT, 1);
- final int emitPerMapTask = (redcount * taskContext.getNumReduceTasks());
+ int reduceTasks = totalIReduces == 0?
+ taskContext.getNumReduceTasks() :
+ conf.getInt(IREDUCE_TASKS_COUNT, 1);
+ int sleepCount = totalIReduces == 0?
+ conf.getInt(REDUCE_SLEEP_COUNT,1) :
+ conf.getInt(IREDUCE_SLEEP_COUNT,1);
+ final int emitPerMapTask = sleepCount * reduceTasks;
return new RecordReader<IntWritable,IntWritable>() {
private int records = 0;
@@ -219,6 +214,9 @@ public class MRRSleepJob extends Configured implements Tool {
) throws IOException, InterruptedException {
//it is expected that every map processes mapSleepCount number of records.
try {
+ LOG.info("Reading in " + vertexName
+ + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+ + " key " + key.get());
LOG.info("Sleeping in InitialMap"
+ ", vertexName=" + vertexName
+ ", taskAttemptId=" + context.getTaskAttemptID()
@@ -228,7 +226,9 @@ public class MRRSleepJob extends Configured implements Tool {
+ (mapSleepDuration * (mapSleepCount - count)));
context.setStatus("Sleeping... (" +
(mapSleepDuration * (mapSleepCount - count)) + ") ms left");
- Thread.sleep(mapSleepDuration);
+ if ((mapSleepCount - count) > 0) {
+ Thread.sleep(mapSleepDuration);
+ }
if (throwError || throwFatal) {
throw new IOException("Throwing a simulated error from map");
}
@@ -242,6 +242,9 @@ public class MRRSleepJob extends Configured implements Tool {
// each reducer will get reduceSleepCount number of keys.
int k = key.get();
for (int i = 0; i < value.get(); ++i) {
+ LOG.info("Writing in " + vertexName
+ + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+ + " key " + (k+i) + " value 1");
context.write(new IntWritable(k + i), new IntWritable(1));
}
}
@@ -269,6 +272,10 @@ public class MRRSleepJob extends Configured implements Tool {
Context context)
throws IOException, InterruptedException {
try {
+ LOG.info("Reading in " + vertexName
+ + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+ + " key " + key.get());
+
LOG.info("Sleeping in IntermediateReduce"
+ ", vertexName=" + vertexName
+ ", taskAttemptId=" + context.getTaskAttemptID()
@@ -278,7 +285,9 @@ public class MRRSleepJob extends Configured implements Tool {
+ (iReduceSleepDuration * (iReduceSleepCount - count)));
context.setStatus("Sleeping... (" +
(iReduceSleepDuration * (iReduceSleepCount - count)) + ") ms left");
- Thread.sleep(iReduceSleepDuration);
+ if ((iReduceSleepCount - count) > 0) {
+ Thread.sleep(iReduceSleepDuration);
+ }
}
catch (InterruptedException ex) {
throw (IOException)new IOException(
@@ -290,6 +299,9 @@ public class MRRSleepJob extends Configured implements Tool {
int k = key.get();
for (IntWritable value : values) {
for (int i = 0; i < value.get(); ++i) {
+ LOG.info("Writing in " + vertexName
+ + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+ + " key " + (k+i) + " value 1");
context.write(new IntWritable(k + i), new IntWritable(1));
}
}
@@ -318,6 +330,9 @@ public class MRRSleepJob extends Configured implements Tool {
Context context)
throws IOException {
try {
+ LOG.info("Reading in " + vertexName
+ + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+ + " key " + key.get());
LOG.info("Sleeping in FinalReduce"
+ ", vertexName=" + vertexName
+ ", taskAttemptId=" + context.getTaskAttemptID()
@@ -327,8 +342,9 @@ public class MRRSleepJob extends Configured implements Tool {
+ (reduceSleepDuration * (reduceSleepCount - count)));
context.setStatus("Sleeping... (" +
(reduceSleepDuration * (reduceSleepCount - count)) + ") ms left");
- Thread.sleep(reduceSleepDuration);
-
+ if ((reduceSleepCount - count) > 0) {
+ Thread.sleep(reduceSleepDuration);
+ }
}
catch (InterruptedException ex) {
throw (IOException)new IOException(
@@ -357,6 +373,7 @@ public class MRRSleepJob extends Configured implements Tool {
conf.setInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
conf.setInt(MRJobConfig.NUM_MAPS, numMapper);
conf.setInt(IREDUCE_STAGES_COUNT, iReduceStagesCount);
+ conf.setInt(IREDUCE_TASKS_COUNT, numIReducer);
// Configure intermediate reduces
conf.setInt(
@@ -364,7 +381,7 @@ public class MRRSleepJob extends Configured implements Tool {
iReduceStagesCount);
LOG.info("Running MRR with " + iReduceStagesCount + " IR stages");
- for (int i = 0; i < iReduceStagesCount; ++i) {
+ for (int i = 1; i <= iReduceStagesCount; ++i) {
// Set reducer class for intermediate reduce
conf.setClass(
MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
[06/50] [abbrv] TEZ-123. Cleanup package dependencies. (bikas)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
index 12ad74c..ccfbd78 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.tez.common.Constants;
import org.apache.tez.common.TezJobConfig;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
/**
* Manipulate the working area for the transient store for maps and reduces.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutput.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutput.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutput.java
index 9f6e10b..eb1fe5f 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutput.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutput.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.fs.Path;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
/**
* Manipulate the working area for the transient store for maps and reduces.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
index eeca130..1bd65e2 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.tez.common.Constants;
import org.apache.tez.common.TezJobConfig;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
/**
* Manipulate the working area for the transient store for maps and reduces.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/tez-mapreduce/pom.xml b/tez-mapreduce/pom.xml
index 2a1ca15..e785f6b 100644
--- a/tez-mapreduce/pom.xml
+++ b/tez-mapreduce/pom.xml
@@ -27,11 +27,15 @@
<dependencies>
<dependency>
<groupId>org.apache.tez</groupId>
- <artifactId>tez-engine</artifactId>
+ <artifactId>tez-dag-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
- <artifactId>tez-dag-api</artifactId>
+ <artifactId>tez-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-engine</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 9dfaf1f..63472d2 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -71,13 +71,13 @@ import org.apache.tez.common.TezTaskStatus;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.common.records.ProceedToCompletionResponse;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.api.Task;
import org.apache.tez.engine.common.task.local.output.TezLocalTaskOutputFiles;
import org.apache.tez.engine.common.task.local.output.TezTaskOutput;
import org.apache.tez.engine.lib.input.LocalMergedInput;
import org.apache.tez.engine.lib.output.LocalOnFileSorterOutput;
import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
import org.apache.tez.engine.runtime.RuntimeUtils;
import org.apache.tez.mapreduce.hadoop.IDConverter;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
index 050f608..b8a99ee 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
import org.apache.tez.common.counters.TaskCounter;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.api.Input;
import org.apache.tez.engine.api.Master;
import org.apache.tez.engine.api.Output;
@@ -46,7 +47,6 @@ import org.apache.tez.engine.common.ConfigUtils;
import org.apache.tez.engine.common.combine.CombineInput;
import org.apache.tez.engine.common.combine.CombineOutput;
import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator;
-import org.apache.tez.engine.records.TezTaskAttemptID;
import org.apache.tez.mapreduce.hadoop.IDConverter;
import org.apache.tez.mapreduce.hadoop.mapred.TaskAttemptContextImpl;
import org.apache.tez.mapreduce.processor.MRTask;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
index 4c56bc4..4046596 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java
@@ -24,10 +24,10 @@ import org.apache.hadoop.mapred.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
public class IDConverter {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRTaskStatus.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRTaskStatus.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRTaskStatus.java
index b5c06eb..e63b704 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRTaskStatus.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRTaskStatus.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapred.TaskStatus;
import org.apache.tez.common.TezTaskStatus;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class MRTaskStatus implements TezTaskStatus {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
index 0b2774d..1fc71b0 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java
@@ -28,9 +28,9 @@ import org.apache.tez.common.TezTaskStatus.Phase;
import org.apache.tez.common.counters.CounterGroup;
import org.apache.tez.common.counters.TezCounter;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
public class TezTypeConverters {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java
index a3e31b0..2f06986 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.Progressable;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.mapreduce.hadoop.IDConverter;
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java
index aacfc60..d953891 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.mapred.TaskID;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.util.Progressable;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.mapreduce.processor.MRTaskReporter;
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
index d27f996..1f34fdb 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.mapreduce.hadoop.IDConverter;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
import org.apache.tez.mapreduce.processor.MRTaskReporter;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
index bd39b67..d7d721f 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java
@@ -63,11 +63,11 @@ import org.apache.tez.common.counters.TaskCounter;
import org.apache.tez.common.counters.TezCounter;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.common.records.ProceedToCompletionResponse;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.api.Master;
import org.apache.tez.engine.common.sort.impl.TezRawKeyValueIterator;
import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
import org.apache.tez.mapreduce.combine.MRCombiner;
import org.apache.tez.mapreduce.hadoop.IDConverter;
import org.apache.tez.mapreduce.hadoop.MRConfig;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/TezTaskReporterImpl.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/TezTaskReporterImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/TezTaskReporterImpl.java
index c601109..d602459 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/TezTaskReporterImpl.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/TezTaskReporterImpl.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
import org.apache.tez.common.counters.TezCounter;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
index 4796646..025bf61 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilicalProtocol.java
@@ -28,8 +28,8 @@ import org.apache.tez.common.ContainerTask;
import org.apache.tez.common.TezTaskStatus;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
import org.apache.tez.common.records.ProceedToCompletionResponse;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
public class TestUmbilicalProtocol implements TezTaskUmbilicalProtocol {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
index 4b04d13..2e823e6 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java
@@ -18,10 +18,10 @@
package org.apache.tez.mapreduce;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
public class TezTestUtils {
[30/50] [abbrv] git commit: TEZ-142. DAG does not throw error when it
receives an unknown DAGSchedulerUpdateEvent (bikas)
Posted by ss...@apache.org.
TEZ-142. DAG does not throw error when it receives an unknown DAGSchedulerUpdateEvent (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a2a72613
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a2a72613
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a2a72613
Branch: refs/heads/master
Commit: a2a726139342c5b510053a0432927a3c5f932666
Parents: 4beb98f
Author: Bikas Saha <bi...@apache.org>
Authored: Fri May 24 17:17:33 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Fri May 24 17:17:33 2013 -0700
----------------------------------------------------------------------
.../org/apache/tez/dag/app/dag/impl/DAGImpl.java | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a2a72613/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 5f47818..4c63d8f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -57,6 +57,7 @@ import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.client.DAGStatusBuilder;
import org.apache.tez.dag.api.client.ProgressBuilder;
@@ -1177,7 +1178,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
dag.dagScheduler.scheduleTask(sEvent);
break;
default:
- LOG.warn("Unknown DAGEventSchedulerUpdate:" + sEvent.getUpdateType());
+ throw new TezException("Unknown DAGEventSchedulerUpdate:"
+ + sEvent.getUpdateType());
}
}
}
[17/50] [abbrv] git commit: TEZ-134. Untangle MR configuration in
YarnTezDagChild. (sseth)
Posted by ss...@apache.org.
TEZ-134. Untangle MR configuration in YarnTezDagChild. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/be6d4bc0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/be6d4bc0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/be6d4bc0
Branch: refs/heads/master
Commit: be6d4bc096d9bae363cbcf1a3dc5ec67b057507d
Parents: 0d5027d
Author: Siddharth Seth <ss...@apache.org>
Authored: Sat May 18 20:51:06 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Sat May 18 20:51:06 2013 -0700
----------------------------------------------------------------------
.../java/org/apache/tez/common/TezJobConfig.java | 19 +-
.../org/apache/hadoop/mapred/YarnTezDagChild.java | 291 ++++-----------
.../main/java/org/apache/tez/engine/api/Task.java | 2 +
.../apache/tez/common/TezEngineTaskContext.java | 5 +
.../tez/engine/common/shuffle/impl/Shuffle.java | 2 +-
.../tez/engine/common/sort/impl/TezMerger.java | 2 +-
.../task/local/output/TezLocalTaskOutputFiles.java | 4 +-
.../task/local/output/TezTaskOutputFiles.java | 2 +-
.../apache/tez/engine/runtime/RuntimeUtils.java | 22 +-
.../org/apache/tez/engine/task/RuntimeTask.java | 21 +-
.../org/apache/hadoop/mapred/LocalJobRunner.java | 8 +-
.../apache/tez/mapreduce/task/MRRuntimeTask.java | 262 +++++++++++++
.../apache/tez/mapreduce/processor/MapUtils.java | 50 +++
.../mapreduce/processor/map/TestMapProcessor.java | 23 +-
.../processor/reduce/TestReduceProcessor.java | 23 +-
15 files changed, 477 insertions(+), 259 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
index 4a7abea..0d6f6be 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -62,11 +62,16 @@ public class TezJobConfig {
public static final long DEFAULT_RECORDS_BEFORE_PROGRESS = 10000;
/**
- *
+ * List of directories avialble to the engine.
*/
- public static final String LOCAL_DIR = "tez.engine.local.dir";
- public static final String DEFAULT_LOCAL_DIR = "/tmp";
-
+ public static final String LOCAL_DIRS = "tez.engine.local.dirs";
+ public static final String DEFAULT_LOCAL_DIRS = "/tmp";
+
+ /**
+ * One local dir for the speicfic job.
+ */
+ public static final String JOB_LOCAL_DIR = "tez.engine.job.local.dir";
+
/**
* The directory which contains the localized files for this task.
*/
@@ -74,6 +79,8 @@ public class TezJobConfig {
public static final String TASK_LOCAL_RESOURCE_DIR = "tez.engine.task-local-resource.dir";
public static final String DEFAULT_TASK_LOCAL_RESOURCE_DIR = "/tmp";
+ public static final String TEZ_TASK_WORKING_DIR = "tez.engine.task.working.dir";
+
/**
*
*/
@@ -299,4 +306,8 @@ public class TezJobConfig {
* credentials.
*/
public static final String DAG_CREDENTIALS_BINARY = "tez.dag.credentials.binary";
+
+
+ public static final String APPLICATION_ATTEMPT_ID =
+ "tez.job.application.attempt.id";
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 2295b6a..6fd6eff 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -20,17 +20,9 @@ package org.apache.hadoop.mapred;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
import java.net.InetSocketAddress;
-import java.net.URI;
import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.crypto.SecretKey;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -39,16 +31,8 @@ import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
-import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
@@ -72,16 +56,15 @@ import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.api.Task;
+import org.apache.tez.engine.common.security.JobTokenIdentifier;
+import org.apache.tez.engine.common.security.TokenCache;
import org.apache.tez.engine.runtime.RuntimeUtils;
-import org.apache.tez.mapreduce.hadoop.DeprecatedKeys;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
-import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+import org.apache.tez.engine.task.RuntimeTask;
import org.apache.tez.mapreduce.input.SimpleInput;
import org.apache.tez.mapreduce.output.SimpleOutput;
-import org.apache.tez.mapreduce.processor.MRTask;
/**
- * The main() for TEZ MapReduce task processes.
+ * The main() for TEZ Task processes.
*/
public class YarnTezDagChild {
@@ -91,11 +74,9 @@ public class YarnTezDagChild {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
LOG.debug("Child starting");
- DeprecatedKeys.init();
-
- final JobConf defaultConf = new JobConf();
- // HACK Eventually load the DagConf for security etc setup.
-// defaultConf.addResource(MRJobConfig.JOB_CONF_FILE);
+ final Configuration defaultConf = new Configuration();
+ // Security settings will be loaded based on core-site and core-default. Don't
+ // depend on the jobConf for this.
UserGroupInformation.setConfiguration(defaultConf);
String host = args[0];
@@ -121,9 +102,10 @@ public class YarnTezDagChild {
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner =
UserGroupInformation.createRemoteUser(appID.toString());
- Token<JobTokenIdentifier> jt = TokenCache.getJobToken(credentials);
- SecurityUtil.setTokenService(jt, address);
- taskOwner.addToken(jt);
+
+ Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(credentials);
+ SecurityUtil.setTokenService(jobToken, address);
+ taskOwner.addToken(jobToken);
final TezTaskUmbilicalProtocol umbilical =
taskOwner.doAs(new PrivilegedExceptionAction<TezTaskUmbilicalProtocol>() {
@Override
@@ -142,9 +124,8 @@ public class YarnTezDagChild {
ContainerTask containerTask = null;
UserGroupInformation childUGI = null;
TezTaskAttemptID taskAttemptId = null;
- MRTask task = null;
ContainerContext containerContext = new ContainerContext(containerId, pid);
-
+
try {
while (true) {
// poll for new task
@@ -165,19 +146,19 @@ public class YarnTezDagChild {
}
taskContext = (TezEngineTaskContext) containerTask
.getTezEngineTaskContext();
- LOG.info("XXXX: New container task context:"
+ LOG.info("DEBUG: New container task context:"
+ taskContext.toString());
taskAttemptId = taskContext.getTaskAttemptId();
final Task t = createAndConfigureTezTask(taskContext, umbilical,
- credentials, jt,
+ credentials, jobToken,
containerId.getApplicationAttemptId().getAttemptId());
- task = (MRTask) t.getProcessor();
- final JobConf job = task.getConf();
- // Initiate Java VM metrics
- JvmMetrics.initSingleton(containerId.toString(), job.getSessionId());
+ final Configuration conf = ((RuntimeTask)t).getConfiguration();
+
+ // TODO Initiate Java VM metrics
+ // JvmMetrics.initSingleton(containerId.toString(), job.getSessionId());
childUGI = UserGroupInformation.createRemoteUser(System
.getenv(ApplicationConstants.Environment.USER.toString()));
// Add tokens to new user so that it may execute its task correctly.
@@ -186,7 +167,7 @@ public class YarnTezDagChild {
childUGI.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
- runTezTask(t, umbilical, job); // run the task
+ runTezTask(t, umbilical, conf); // run the task
return null;
}
});
@@ -196,34 +177,6 @@ public class YarnTezDagChild {
} catch (FSError e) {
LOG.fatal("FSError from child", e);
umbilical.fsError(taskAttemptId, e.getMessage());
- } catch (Exception exception) {
- LOG.warn("Exception running child : "
- + StringUtils.stringifyException(exception));
- try {
- if (task != null) {
- // do cleanup for the task
- if (childUGI == null) { // no need to job into doAs block
- task.taskCleanup(umbilical);
- } else {
- final MRTask taskFinal = task;
- childUGI.doAs(new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- taskFinal.taskCleanup(umbilical);
- return null;
- }
- });
- }
- }
- } catch (Exception e) {
- LOG.info("Exception cleaning up: " + StringUtils.stringifyException(e));
- }
- // Report back any failures, for diagnostic purposes
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- exception.printStackTrace(new PrintStream(baos));
- if (taskAttemptId != null) {
- umbilical.fatalError(taskAttemptId, baos.toString());
- }
} catch (Throwable throwable) {
LOG.fatal("Error running child : "
+ StringUtils.stringifyException(throwable));
@@ -249,29 +202,36 @@ public class YarnTezDagChild {
* out an output directory.
* @throws IOException
*/
- private static void configureLocalDirs(JobConf job) throws IOException {
+ /**
+ * Configure tez-local-dirs, tez-localized-file-dir, etc. Also create these
+ * dirs.
+ */
+
+ private static void configureLocalDirs(Configuration conf) throws IOException {
String[] localSysDirs = StringUtils.getTrimmedStrings(
System.getenv(Environment.LOCAL_DIRS.name()));
- job.setStrings(TezJobConfig.LOCAL_DIR, localSysDirs);
- job.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
+ conf.setStrings(TezJobConfig.LOCAL_DIRS, localSysDirs);
+ conf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
System.getenv(Environment.PWD.name()));
- LOG.info(TezJobConfig.LOCAL_DIR + " for child: " +
- job.get(TezJobConfig.LOCAL_DIR));
+
+ LOG.info(TezJobConfig.LOCAL_DIRS + " for child: " +
+ conf.get(TezJobConfig.LOCAL_DIRS));
LOG.info(TezJobConfig.TASK_LOCAL_RESOURCE_DIR + " for child: "
- + job.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
- LocalDirAllocator lDirAlloc = new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
+ + conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
+
+ LocalDirAllocator lDirAlloc = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
Path workDir = null;
// First, try to find the JOB_LOCAL_DIR on this host.
try {
- workDir = lDirAlloc.getLocalPathToRead("work", job);
+ workDir = lDirAlloc.getLocalPathToRead("work", conf);
} catch (DiskErrorException e) {
// DiskErrorException means dir not found. If not found, it will
// be created below.
}
if (workDir == null) {
// JOB_LOCAL_DIR doesn't exist on this host -- Create it.
- workDir = lDirAlloc.getLocalPathForWrite("work", job);
- FileSystem lfs = FileSystem.getLocal(job).getRaw();
+ workDir = lDirAlloc.getLocalPathForWrite("work", conf);
+ FileSystem lfs = FileSystem.getLocal(conf).getRaw();
boolean madeDir = false;
try {
madeDir = lfs.mkdirs(workDir);
@@ -281,155 +241,29 @@ public class YarnTezDagChild {
// at the same time. If this task loses the race, it's okay because
// the directory already exists.
madeDir = true;
- workDir = lDirAlloc.getLocalPathToRead("work", job);
+ workDir = lDirAlloc.getLocalPathToRead("work", conf);
}
if (!madeDir) {
throw new IOException("Mkdirs failed to create "
+ workDir.toString());
}
}
- // TODO TEZ This likely needs fixing to make sure things work when there are multiple local-dirs etc.
- job.set(MRJobConfig.JOB_LOCAL_DIR, workDir.toString());
- }
-
- private static JobConf configureTask(MRTask task, Credentials credentials,
- Token<JobTokenIdentifier> jt, int appAttemptId)
- throws IOException, InterruptedException {
- JobConf job = task.getConf();
-
- // Set it in conf, so as to be able to be used the the OutputCommitter.
- job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId);
-
- // set tcp nodelay
- job.setBoolean("ipc.client.tcpnodelay", true);
- job.setClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
- YarnOutputFiles.class, MapOutputFile.class);
- // set the jobTokenFile into task
- SecretKey sk = JobTokenSecretManager.createSecretKey(jt.getPassword());
-
- task.setJobTokenSecret(sk);
-// task.setJobTokenSecret(
-// JobTokenSecretManager.createSecretKey(jt.getPassword()));
-
- // setup the child's MRConfig.LOCAL_DIR.
- configureLocalDirs(job);
-
- // setup the child's attempt directories
- // Do the task-type specific localization
- task.localizeConfiguration(job);
-
- // Set up the DistributedCache related configs
- setupDistributedCacheConfig(job);
-
- // Overwrite the localized task jobconf which is linked to in the current
- // work-dir.
- Path localTaskFile = new Path(MRJobConfig.JOB_CONF_FILE);
- writeLocalJobFile(localTaskFile, job);
- task.setConf(job);
- return job;
- }
-
- /**
- * Set up the DistributedCache related configs to make
- * {@link DistributedCache#getLocalCacheFiles(Configuration)}
- * and
- * {@link DistributedCache#getLocalCacheArchives(Configuration)}
- * working.
- * @param job
- * @throws IOException
- */
- private static void setupDistributedCacheConfig(final JobConf job)
- throws IOException {
-
- String localWorkDir = System.getenv("PWD");
- // ^ ^ all symlinks are created in the current work-dir
-
- // Update the configuration object with localized archives.
- URI[] cacheArchives = DistributedCache.getCacheArchives(job);
- if (cacheArchives != null) {
- List<String> localArchives = new ArrayList<String>();
- for (int i = 0; i < cacheArchives.length; ++i) {
- URI u = cacheArchives[i];
- Path p = new Path(u);
- Path name =
- new Path((null == u.getFragment()) ? p.getName()
- : u.getFragment());
- String linkName = name.toUri().getPath();
- localArchives.add(new Path(localWorkDir, linkName).toUri().getPath());
- }
- if (!localArchives.isEmpty()) {
- job.set(MRJobConfig.CACHE_LOCALARCHIVES, StringUtils
- .arrayToString(localArchives.toArray(new String[localArchives
- .size()])));
- }
- }
-
- // Update the configuration object with localized files.
- URI[] cacheFiles = DistributedCache.getCacheFiles(job);
- if (cacheFiles != null) {
- List<String> localFiles = new ArrayList<String>();
- for (int i = 0; i < cacheFiles.length; ++i) {
- URI u = cacheFiles[i];
- Path p = new Path(u);
- Path name =
- new Path((null == u.getFragment()) ? p.getName()
- : u.getFragment());
- String linkName = name.toUri().getPath();
- localFiles.add(new Path(localWorkDir, linkName).toUri().getPath());
- }
- if (!localFiles.isEmpty()) {
- job.set(MRJobConfig.CACHE_LOCALFILES,
- StringUtils.arrayToString(localFiles
- .toArray(new String[localFiles.size()])));
- }
- }
- }
-
- private static final FsPermission urw_gr =
- FsPermission.createImmutable((short) 0640);
-
- /**
- * Write the task specific job-configuration file.
- * @throws IOException
- */
- private static void writeLocalJobFile(Path jobFile, JobConf conf)
- throws IOException {
- FileSystem localFs = FileSystem.getLocal(conf);
- localFs.delete(jobFile);
- OutputStream out = null;
- try {
- out = FileSystem.create(localFs, jobFile, urw_gr);
- conf.writeXml(out);
- } finally {
- IOUtils.cleanup(LOG, out);
- }
+ conf.set(TezJobConfig.JOB_LOCAL_DIR, workDir.toString());
}
-
+
private static Task createAndConfigureTezTask(
- TezEngineTaskContext taskContext,
- TezTaskUmbilicalProtocol master,
- Credentials credentials, Token<JobTokenIdentifier> jt,
- int appAttemptId)
- throws IOException, InterruptedException {
+ TezEngineTaskContext taskContext, TezTaskUmbilicalProtocol master,
+ Credentials cxredentials, Token<JobTokenIdentifier> jobToken,
+ int appAttemptId) throws IOException, InterruptedException {
- Configuration jConf = new JobConf(MRJobConfig.JOB_CONF_FILE);
- Configuration conf = MultiStageMRConfigUtil.getConfForVertex(jConf,
- taskContext.getVertexName());
+ Configuration conf = new Configuration();
+ // set tcp nodelay
+ conf.setBoolean("ipc.client.tcpnodelay", true);
+ conf.setInt(TezJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId);
+
+ configureLocalDirs(conf);
+
- // TOOD Post MRR
- // A single file per vertex will likely be a better solution. Does not
- // require translation - client can take care of this. Will work independent
- // of whether the configuration is for intermediate tasks or not. Has the
- // overhead of localizing multiple files per job - i.e. the client would
- // need to write these files to hdfs, add them as local resources per
- // vertex. A solution like this may be more practical once it's possible to
- // submit configuration parameters to the AM and effectively tasks via RPC.
-
- // TODO Avoid all this extra config manipulation.
- // FIXME we need I/O/p level configs to be used in init below
- final JobConf job = new JobConf(conf);
- job.setCredentials(credentials);
-
// FIXME need Input/Output vertices else we have this hack
if (taskContext.getInputSpecList().isEmpty()) {
taskContext.getInputSpecList().add(
@@ -440,26 +274,37 @@ public class YarnTezDagChild {
new OutputSpec("null", 0, SimpleOutput.class.getName()));
}
Task t = RuntimeUtils.createRuntimeTask(taskContext);
-
+ t.initialize(conf, master);
+
// FIXME wrapper should initialize all of processor, inputs and outputs
// Currently, processor is inited via task init
// and processor then inits inputs and outputs
- t.initialize(job, master);
-
- MRTask task = (MRTask)t.getProcessor();
- configureTask(task, credentials, jt, appAttemptId);
-
return t;
}
private static void runTezTask(
- Task t, TezTaskUmbilicalProtocol master, JobConf job)
+ Task t, TezTaskUmbilicalProtocol master, Configuration conf)
throws IOException, InterruptedException {
// use job-specified working directory
- FileSystem.get(job).setWorkingDirectory(job.getWorkingDirectory());
+ FileSystem.get(conf).setWorkingDirectory(getWorkingDirectory(conf));
// Run!
t.run();
t.close();
}
+
+ private static Path getWorkingDirectory(Configuration conf) {
+ String name = conf.get(JobContext.WORKING_DIR);
+ if (name != null) {
+ return new Path(name);
+ } else {
+ try {
+ Path dir = FileSystem.get(conf).getWorkingDirectory();
+ conf.set(JobContext.WORKING_DIR, dir.toString());
+ return dir;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine-api/src/main/java/org/apache/tez/engine/api/Task.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/api/Task.java b/tez-engine-api/src/main/java/org/apache/tez/engine/api/Task.java
index ea50a06..c08eda6 100644
--- a/tez-engine-api/src/main/java/org/apache/tez/engine/api/Task.java
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/api/Task.java
@@ -74,4 +74,6 @@ public interface Task {
*/
public void close() throws IOException, InterruptedException;
+
+ public Configuration getConfiguration();
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
index 5823de6..93aeb0b 100644
--- a/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
+++ b/tez-engine/src/main/java/org/apache/tez/common/TezEngineTaskContext.java
@@ -55,6 +55,11 @@ public class TezEngineTaskContext extends TezTaskContext {
this.processorName = processorName;
}
+ public String getRuntimeName() {
+ // FIXME. Add this to the DAG configuration, and fetch from there.
+ return "org.apache.tez.mapreduce.task.MRRuntimeTask";
+ }
+
public String getProcessorName() {
return processorName;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java
index 6a065de..202abd6 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/impl/Shuffle.java
@@ -80,7 +80,7 @@ public class Shuffle implements ExceptionReporter {
FileSystem localFS = FileSystem.getLocal(this.conf);
LocalDirAllocator localDirAllocator =
- new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
+ new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
copyPhase = this.runningTaskContext.getProgress().addPhase("copy", 0.33f);
mergePhase = this.runningTaskContext.getProgress().addPhase("merge", 0.66f);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/TezMerger.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/TezMerger.java b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/TezMerger.java
index 829495b..eeea764 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/TezMerger.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/sort/impl/TezMerger.java
@@ -57,7 +57,7 @@ public class TezMerger {
// Local directories
private static LocalDirAllocator lDirAlloc =
- new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
+ new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
public static
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
index ccfbd78..69484af 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezLocalTaskOutputFiles.java
@@ -42,7 +42,7 @@ import org.apache.tez.dag.records.TezTaskID;
public class TezLocalTaskOutputFiles extends TezTaskOutput {
private LocalDirAllocator lDirAlloc =
- new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
+ new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
public TezLocalTaskOutputFiles() {
}
@@ -223,7 +223,7 @@ public class TezLocalTaskOutputFiles extends TezTaskOutput {
}
private String[] getLocalDirs() throws IOException {
- return getConf().getStrings(TezJobConfig.LOCAL_DIR);
+ return getConf().getStrings(TezJobConfig.LOCAL_DIRS);
}
@SuppressWarnings("deprecation")
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
index 1bd65e2..5fb6519 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/task/local/output/TezTaskOutputFiles.java
@@ -56,7 +56,7 @@ public class TezTaskOutputFiles extends TezTaskOutput {
// assume configured to $localdir/usercache/$user/appcache/$appId
private LocalDirAllocator lDirAlloc =
- new LocalDirAllocator(TezJobConfig.LOCAL_DIR);
+ new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
private Path getAttemptOutputDir() {
LOG.info("DEBUG: getAttemptOutputDir: "
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
index e04d405..900c2f0 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/runtime/RuntimeUtils.java
@@ -66,6 +66,7 @@ public class RuntimeUtils {
+ ", Processor: " + taskContext.getProcessorName()
+ ", InputCount=" + taskContext.getInputSpecList().size()
+ ", OutputCount=" + taskContext.getOutputSpecList().size());
+
RuntimeTask t = null;
try {
Class<?> processorClazz =
@@ -108,11 +109,30 @@ public class RuntimeUtils {
outputs[i] = output;
}
}
- t = new RuntimeTask(processor, inputs, outputs);
+ // t = new RuntimeTask(taskContext, processor, inputs, outputs);
+ t = createRuntime(taskContext, processor, inputs, outputs);
} catch (ClassNotFoundException e) {
throw new YarnException("Unable to initialize RuntimeTask, context="
+ taskContext, e);
}
return t;
}
+
+ private static RuntimeTask createRuntime(TezEngineTaskContext taskContext,
+ Processor processor, Input[] inputs, Output[] outputs) {
+ try {
+ // TODO Change this to use getNewInstance
+ Class<?> runtimeClazz = Class.forName(taskContext.getRuntimeName());
+ Constructor<?> ctor = runtimeClazz.getConstructor(
+ TezEngineTaskContext.class, Processor.class, Input[].class,
+ Output[].class);
+ ctor.setAccessible(true);
+ return (RuntimeTask) ctor.newInstance(taskContext, processor, inputs, outputs);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Unable to load runtimeClass: "
+ + taskContext.getRuntimeName(), e);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-engine/src/main/java/org/apache/tez/engine/task/RuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/task/RuntimeTask.java b/tez-engine/src/main/java/org/apache/tez/engine/task/RuntimeTask.java
index 3e2c6f2..871f3ba 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/task/RuntimeTask.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/task/RuntimeTask.java
@@ -20,6 +20,7 @@ package org.apache.tez.engine.task;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.TezEngineTaskContext;
import org.apache.tez.engine.api.Input;
import org.apache.tez.engine.api.Master;
import org.apache.tez.engine.api.Output;
@@ -28,17 +29,19 @@ import org.apache.tez.engine.api.Task;
public class RuntimeTask implements Task {
- private final Input[] inputs;
- private final Output[] outputs;
- private final Processor processor;
+ protected final Input[] inputs;
+ protected final Output[] outputs;
+ protected final Processor processor;
- private Configuration conf;
- private Master master;
+ protected TezEngineTaskContext taskContext;
+ protected Configuration conf;
+ protected Master master;
- public RuntimeTask(
+ public RuntimeTask(TezEngineTaskContext taskContext,
Processor processor,
Input[] inputs,
Output[] outputs) {
+ this.taskContext = taskContext;
this.inputs = inputs;
this.processor = processor;
this.outputs = outputs;
@@ -74,7 +77,13 @@ public class RuntimeTask implements Task {
public void close() throws IOException, InterruptedException {
// NOTE: Allow processor to close input/output
+ // This can be changed to close input/output since MRRuntimeTask is used for
+ // MR jobs, which changes the order.
processor.close();
}
+ @Override
+ public Configuration getConfiguration() {
+ return this.conf;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 63472d2..5c181e2 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -465,7 +465,7 @@ public class LocalJobRunner implements ClientProtocol {
LOG.info("XXX mapId: " + i +
" LOCAL_DIR = " +
mapOutputFiles.get(mapId).getConf().get(
- TezJobConfig.LOCAL_DIR));
+ TezJobConfig.LOCAL_DIRS));
Path mapOut = mapOutputFiles.get(mapId).getOutputFile();
TezTaskOutput localOutputFile = new TezLocalTaskOutputFiles();
localOutputFile.setConf(localConf);
@@ -902,7 +902,7 @@ public class LocalJobRunner implements ClientProtocol {
TaskAttemptID taskAttemptID, String user, JobConf conf) {
String[] localDirs =
conf.getTrimmedStrings(
- TezJobConfig.LOCAL_DIR, TezJobConfig.DEFAULT_LOCAL_DIR);
+ TezJobConfig.LOCAL_DIRS, TezJobConfig.DEFAULT_LOCAL_DIRS);
String jobId = taskAttemptID.getJobID().toString();
String taskId = taskAttemptID.getTaskID().toString();
boolean isCleanup = false;
@@ -913,9 +913,9 @@ public class LocalJobRunner implements ClientProtocol {
childMapredLocalDir.append("," + localDirs[i] + Path.SEPARATOR
+ getLocalTaskDir(user, jobId, taskId, isCleanup));
}
- LOG.info(TezJobConfig.LOCAL_DIR + " for child : " + taskAttemptID +
+ LOG.info(TezJobConfig.LOCAL_DIRS + " for child : " + taskAttemptID +
" is " + childMapredLocalDir);
- conf.set(TezJobConfig.LOCAL_DIR, childMapredLocalDir.toString());
+ conf.set(TezJobConfig.LOCAL_DIRS, childMapredLocalDir.toString());
conf.setClass(Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
TezLocalTaskOutputFiles.class, TezTaskOutput.class);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
new file mode 100644
index 0000000..6091d4b
--- /dev/null
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -0,0 +1,262 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.task;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.crypto.SecretKey;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSError;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapOutputFile;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.filecache.DistributedCache;
+import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
+import org.apache.tez.common.TezEngineTaskContext;
+import org.apache.tez.common.TezJobConfig;
+import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.engine.api.Input;
+import org.apache.tez.engine.api.Master;
+import org.apache.tez.engine.api.Output;
+import org.apache.tez.engine.api.Processor;
+import org.apache.tez.engine.common.security.JobTokenIdentifier;
+import org.apache.tez.engine.common.security.TokenCache;
+import org.apache.tez.engine.task.RuntimeTask;
+import org.apache.tez.mapreduce.hadoop.DeprecatedKeys;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+import org.apache.tez.mapreduce.processor.MRTask;
+import org.apache.tez.mapreduce.task.impl.YarnOutputFiles;
+
+@SuppressWarnings("deprecation")
+public class MRRuntimeTask extends RuntimeTask {
+
+ private static final Log LOG = LogFactory.getLog(MRRuntimeTask.class);
+
+ private MRTask mrTask;
+
+ public MRRuntimeTask(TezEngineTaskContext taskContext, Processor processor,
+ Input[] inputs, Output[] outputs) {
+ super(taskContext, processor, inputs, outputs);
+ }
+
+ @Override
+ public void initialize(Configuration conf, Master master) throws IOException,
+ InterruptedException {
+
+ DeprecatedKeys.init();
+
+ Configuration mrConf = new Configuration(conf);
+ mrConf.addResource(MRJobConfig.JOB_CONF_FILE);
+ Configuration taskConf = MultiStageMRConfigUtil.getConfForVertex(mrConf,
+ taskContext.getVertexName());
+
+ // TODO Avoid all this extra config manipulation.
+ // FIXME we need I/O/p level configs to be used in init below
+
+ // TOOD Post MRR
+ // A single file per vertex will likely be a better solution. Does not
+ // require translation - client can take care of this. Will work independent
+ // of whether the configuration is for intermediate tasks or not. Has the
+ // overhead of localizing multiple files per job - i.e. the client would
+ // need to write these files to hdfs, add them as local resources per
+ // vertex. A solution like this may be more practical once it's possible to
+ // submit configuration parameters to the AM and effectively tasks via RPC.
+
+ final JobConf job = new JobConf(taskConf);
+
+ MRTask mrTask = (MRTask) getProcessor();
+ this.mrTask = mrTask;
+ configureMRTask(job, mrTask);
+
+ this.conf = job;
+ this.master = master;
+
+ // NOTE: Allow processor to initialize input/output
+ processor.initialize(this.conf, this.master);
+ }
+
+ @Override
+ public void run() throws IOException, InterruptedException {
+ TezTaskUmbilicalProtocol umbilical = (TezTaskUmbilicalProtocol) master;
+ try {
+ super.run();
+ } catch (FSError e) {
+ throw e;
+ } catch (Exception exception) {
+ LOG.warn("Exception running child : "
+ + StringUtils.stringifyException(exception));
+ try {
+ if (mrTask != null) {
+ mrTask.taskCleanup(umbilical);
+ }
+ } catch (Exception e) {
+ LOG.info("Exception cleanup up: " + StringUtils.stringifyException(e));
+ }
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ exception.printStackTrace(new PrintStream(baos));
+ if (taskContext.getTaskAttemptId() != null) {
+ umbilical.fatalError(taskContext.getTaskAttemptId(), baos.toString());
+ }
+ }
+ }
+
+ @Override
+ public void close() throws IOException, InterruptedException {
+ // NOTE: Allow processor to close input/output
+ processor.close();
+ }
+
+ private static void configureMRTask(JobConf job, MRTask task)
+ throws IOException, InterruptedException {
+
+ Credentials credentials = UserGroupInformation.getCurrentUser()
+ .getCredentials();
+ job.setCredentials(credentials);
+ // TODO Can this be avoided all together. Have the MRTezOutputCommitter use
+ // the Tez parameter.
+ // TODO This could be fetched from the env if YARN is setting it for all
+ // Containers.
+ // Set it in conf, so as to be able to be used the the OutputCommitter.
+ job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
+ job.getInt(TezJobConfig.APPLICATION_ATTEMPT_ID, -1));
+
+ job.setClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS, YarnOutputFiles.class,
+ MapOutputFile.class); // MR
+
+ Token<JobTokenIdentifier> jobToken = TokenCache.getJobToken(credentials);
+ if (jobToken != null) {
+ // Will MR ever run without a job token.
+ SecretKey sk = JobTokenSecretManager.createSecretKey(jobToken
+ .getPassword());
+ task.setJobTokenSecret(sk);
+ } else {
+ LOG.warn("No job token set");
+ }
+
+ job.set(MRJobConfig.JOB_LOCAL_DIR, job.get(TezJobConfig.JOB_LOCAL_DIR));
+
+ // setup the child's attempt directories
+ // Do the task-type specific localization
+ task.localizeConfiguration(job);
+
+ // Set up the DistributedCache related configs
+ setupDistributedCacheConfig(job);
+
+ // Overwrite the localized task jobconf which is linked to in the current
+ // work-dir.
+ Path localTaskFile = new Path(
+ job.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR),
+ MRJobConfig.JOB_CONF_FILE);
+ writeLocalJobFile(localTaskFile, job);
+
+ task.setConf(job);
+ }
+
+ /**
+ * Set up the DistributedCache related configs to make
+ * {@link DistributedCache#getLocalCacheFiles(Configuration)} and
+ * {@link DistributedCache#getLocalCacheArchives(Configuration)} working.
+ *
+ * @param job
+ * @throws IOException
+ */
+ private static void setupDistributedCacheConfig(final JobConf job)
+ throws IOException {
+
+ String localWorkDir = System.getenv(System.getenv(Environment.PWD.name()));
+ // ^ ^ all symlinks are created in the current work-dir
+
+ // Update the configuration object with localized archives.
+ URI[] cacheArchives = DistributedCache.getCacheArchives(job);
+ if (cacheArchives != null) {
+ List<String> localArchives = new ArrayList<String>();
+ for (int i = 0; i < cacheArchives.length; ++i) {
+ URI u = cacheArchives[i];
+ Path p = new Path(u);
+ Path name = new Path((null == u.getFragment()) ? p.getName()
+ : u.getFragment());
+ String linkName = name.toUri().getPath();
+ localArchives.add(new Path(localWorkDir, linkName).toUri().getPath());
+ }
+ if (!localArchives.isEmpty()) {
+ job.set(MRJobConfig.CACHE_LOCALARCHIVES, StringUtils
+ .arrayToString(localArchives.toArray(new String[localArchives
+ .size()])));
+ }
+ }
+
+ // Update the configuration object with localized files.
+ URI[] cacheFiles = DistributedCache.getCacheFiles(job);
+ if (cacheFiles != null) {
+ List<String> localFiles = new ArrayList<String>();
+ for (int i = 0; i < cacheFiles.length; ++i) {
+ URI u = cacheFiles[i];
+ Path p = new Path(u);
+ Path name = new Path((null == u.getFragment()) ? p.getName()
+ : u.getFragment());
+ String linkName = name.toUri().getPath();
+ localFiles.add(new Path(localWorkDir, linkName).toUri().getPath());
+ }
+ if (!localFiles.isEmpty()) {
+ job.set(MRJobConfig.CACHE_LOCALFILES, StringUtils
+ .arrayToString(localFiles.toArray(new String[localFiles.size()])));
+ }
+ }
+ }
+
+ private static final FsPermission urw_gr = FsPermission
+ .createImmutable((short) 0640);
+
+ /**
+ * Write the task specific job-configuration file.
+ *
+ * @throws IOException
+ */
+ private static void writeLocalJobFile(Path jobFile, JobConf conf)
+ throws IOException {
+ FileSystem localFs = FileSystem.getLocal(conf);
+ localFs.delete(jobFile);
+ OutputStream out = null;
+ try {
+ out = FileSystem.create(localFs, jobFile, urw_gr);
+ conf.writeXml(out);
+ } finally {
+ IOUtils.cleanup(LOG, out);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
index 0b6bc5f..48a1113 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
@@ -24,14 +24,17 @@ import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
@@ -40,6 +43,8 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.split.JobSplit;
import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo;
import org.apache.hadoop.mapreduce.split.SplitMetaInfoReaderTez;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.tez.common.InputSpec;
import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.TezEngineTaskContext;
@@ -55,6 +60,51 @@ public class MapUtils {
private static final Log LOG = LogFactory.getLog(MapUtils.class);
+ public static void configureLocalDirs(Configuration conf, String localDir)
+ throws IOException {
+ String[] localSysDirs = new String[1];
+ localSysDirs[0] = localDir;
+
+ conf.setStrings(TezJobConfig.LOCAL_DIRS, localSysDirs);
+ conf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
+ System.getenv(Environment.PWD.name()));
+
+ LOG.info(TezJobConfig.LOCAL_DIRS + " for child: "
+ + conf.get(TezJobConfig.LOCAL_DIRS));
+ LOG.info(TezJobConfig.TASK_LOCAL_RESOURCE_DIR + " for child: "
+ + conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
+
+ LocalDirAllocator lDirAlloc = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
+ Path workDir = null;
+ // First, try to find the JOB_LOCAL_DIR on this host.
+ try {
+ workDir = lDirAlloc.getLocalPathToRead("work", conf);
+ } catch (DiskErrorException e) {
+ // DiskErrorException means dir not found. If not found, it will
+ // be created below.
+ }
+ if (workDir == null) {
+ // JOB_LOCAL_DIR doesn't exist on this host -- Create it.
+ workDir = lDirAlloc.getLocalPathForWrite("work", conf);
+ FileSystem lfs = FileSystem.getLocal(conf).getRaw();
+ boolean madeDir = false;
+ try {
+ madeDir = lfs.mkdirs(workDir);
+ } catch (FileAlreadyExistsException e) {
+ // Since all tasks will be running in their own JVM, the race condition
+ // exists where multiple tasks could be trying to create this directory
+ // at the same time. If this task loses the race, it's okay because
+ // the directory already exists.
+ madeDir = true;
+ workDir = lDirAlloc.getLocalPathToRead("work", conf);
+ }
+ if (!madeDir) {
+ throw new IOException("Mkdirs failed to create " + workDir.toString());
+ }
+ }
+ conf.set(TezJobConfig.JOB_LOCAL_DIR, workDir.toString());
+ }
+
private static InputSplit
createInputSplit(FileSystem fs, Path workDir, JobConf job, Path file)
throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
index cda15fb..1e5fe9b 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
@@ -55,36 +55,40 @@ import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
+@SuppressWarnings("deprecation")
public class TestMapProcessor {
private static final Log LOG = LogFactory.getLog(TestMapProcessor.class);
private static JobConf defaultConf = new JobConf();
private static FileSystem localFs = null;
+ private static Path workDir = null;
static {
try {
defaultConf.set("fs.defaultFS", "file:///");
localFs = FileSystem.getLocal(defaultConf);
+ workDir =
+ new Path(new Path(System.getProperty("test.build.data", "/tmp")),
+ "TestMapProcessor").makeQualified(localFs);
+ MapUtils.configureLocalDirs(defaultConf, workDir.toString());
} catch (IOException e) {
throw new RuntimeException("init failure", e);
}
}
- @SuppressWarnings("deprecation")
- private static Path workDir =
- new Path(new Path(System.getProperty("test.build.data", "/tmp")),
- "TestMapProcessor").makeQualified(localFs);
+
+
TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
public void setUpJobConf(JobConf job) {
- job.set(TezJobConfig.LOCAL_DIR, workDir.toString());
+ job.set(TezJobConfig.LOCAL_DIRS, workDir.toString());
job.setClass(
Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
TezLocalTaskOutputFiles.class,
TezTaskOutput.class);
job.setNumReduceTasks(1);
}
-
+
@Before
@After
public void cleanup() throws Exception {
@@ -98,15 +102,18 @@ public class TestMapProcessor {
setUpJobConf(jobConf);
TezTaskOutput mapOutputs = new TezLocalTaskOutputFiles();
mapOutputs.setConf(jobConf);
-
+
Configuration conf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
+ conf.setInt(TezJobConfig.APPLICATION_ATTEMPT_ID, 0);
+
Configuration stageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
vertexName);
JobConf job = new JobConf(stageConf);
-
+
job.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
"localized-resources").toUri().toString());
+
MapUtils.runMapProcessor(localFs, workDir, job, 0,
new Path(workDir, "map0"), new TestUmbilicalProtocol(), vertexName,
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/be6d4bc0/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
index 69571e1..d17b0be 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
@@ -56,27 +56,30 @@ import org.junit.Before;
import org.junit.Test;
+@SuppressWarnings("deprecation")
public class TestReduceProcessor {
private static final Log LOG = LogFactory.getLog(TestReduceProcessor.class);
private static JobConf defaultConf = new JobConf();
- private static FileSystem localFs = null;
+ private static FileSystem localFs = null;
+ private static Path workDir = null;
static {
try {
defaultConf.set("fs.defaultFS", "file:///");
localFs = FileSystem.getLocal(defaultConf);
+ workDir =
+ new Path(new Path(System.getProperty("test.build.data", "/tmp")),
+ "TestReduceProcessor").makeQualified(localFs);
+
+ MapUtils.configureLocalDirs(defaultConf, workDir.toString());
} catch (IOException e) {
throw new RuntimeException("init failure", e);
}
}
- @SuppressWarnings("deprecation")
- private static Path workDir =
- new Path(new Path(System.getProperty("test.build.data", "/tmp")),
- "TestReduceProcessor").makeQualified(localFs);
public void setUpJobConf(JobConf job) {
- job.set(TezJobConfig.LOCAL_DIR, workDir.toString());
+ job.set(TezJobConfig.LOCAL_DIRS, workDir.toString());
job.setClass(
Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER,
TezLocalTaskOutputFiles.class,
@@ -101,6 +104,7 @@ public class TestReduceProcessor {
mapOutputs.setConf(jobConf);
Configuration conf = MultiStageMRConfToTezTranslator.convertMRToLinearTez(jobConf);
+ conf.setInt(TezJobConfig.APPLICATION_ATTEMPT_ID, 0);
Configuration mapStageConf = MultiStageMRConfigUtil.getConfForVertex(conf,
mapVertexName);
@@ -109,6 +113,7 @@ public class TestReduceProcessor {
mapConf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
"localized-resources").toUri().toString());
+
// Run a map
MapUtils.runMapProcessor(localFs, workDir, mapConf, 0,
new Path(workDir, "map0"), new TestUmbilicalProtocol(), mapVertexName,
@@ -124,6 +129,8 @@ public class TestReduceProcessor {
reduceVertexName);
JobConf reduceConf = new JobConf(reduceStageConf);
reduceConf.setOutputFormat(SequenceFileOutputFormat.class);
+ reduceConf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
+ "localized-resources").toUri().toString());
FileOutputFormat.setOutputPath(reduceConf, new Path(workDir, "output"));
// Now run a reduce
@@ -147,8 +154,8 @@ public class TestReduceProcessor {
.toMRTaskId(taskContext.getTaskAttemptId().getTaskID()));
Path reduceOutputFile = new Path(reduceOutputDir, "part-00000");
- @SuppressWarnings("deprecation")
- SequenceFile.Reader reader = new SequenceFile.Reader(localFs, reduceOutputFile, reduceConf);
+ SequenceFile.Reader reader = new SequenceFile.Reader(localFs,
+ reduceOutputFile, reduceConf);
LongWritable key = new LongWritable();
Text value = new Text();
[26/50] [abbrv] git commit: TEZ-150. Fix usage of mapreduce API keys
for num maps and num reduces. Removes Tez configuration for in-degree and
out-degree. (sseth)
Posted by ss...@apache.org.
TEZ-150. Fix usage of mapreduce API keys for num maps and num reduces.
Removes Tez configuration for in-degree and out-degree. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/4c2732c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/4c2732c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/4c2732c9
Branch: refs/heads/master
Commit: 4c2732c991a4cc76011acedbf9c17726c06bc145
Parents: 268f3ac
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu May 23 13:40:22 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu May 23 13:40:22 2013 -0700
----------------------------------------------------------------------
.../java/org/apache/tez/common/TezJobConfig.java | 16 +---------
.../org/apache/hadoop/mapred/LocalJobRunner.java | 1 -
.../tez/mapreduce/hadoop/DeprecatedKeys.java | 17 ++++------
.../processor/reduce/ReduceProcessor.java | 1 -
.../apache/tez/mapreduce/task/MRRuntimeTask.java | 4 ++
.../tez/mapreduce/hadoop/TestDeprecatedKeys.java | 23 +-------------
6 files changed, 14 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
index 0d6f6be..c0b0ae6 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -66,7 +66,7 @@ public class TezJobConfig {
*/
public static final String LOCAL_DIRS = "tez.engine.local.dirs";
public static final String DEFAULT_LOCAL_DIRS = "/tmp";
-
+
/**
* One local dir for the speicfic job.
*/
@@ -80,20 +80,6 @@ public class TezJobConfig {
public static final String DEFAULT_TASK_LOCAL_RESOURCE_DIR = "/tmp";
public static final String TEZ_TASK_WORKING_DIR = "tez.engine.task.working.dir";
-
- /**
- *
- */
- public static final String TEZ_ENGINE_TASK_INDEGREE =
- "tez.engine.task.in-degree";
- public static final int DEFAULT_TEZ_ENGINE_TASK_INDEGREE = 1;
-
- /**
- *
- */
- public static final String TEZ_ENGINE_TASK_OUTDEGREE =
- "tez.engine.task.out-degree";
- public static final int DEFAULT_TEZ_ENGINE_TASK_OUTDEGREE = 1;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 5c181e2..5dade2b 100644
--- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -445,7 +445,6 @@ public class LocalJobRunner implements ClientProtocol {
JobConf localConf = new JobConf(job);
localConf.setUser(user);
localConf.set("mapreduce.jobtracker.address", "local");
- localConf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, mapIds.size());
setupChildMapredLocalDirs(reduceId, user, localConf);
// FIXME invalid task context
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index 01d4fe0..499159f 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -136,8 +136,6 @@ public class DeprecatedKeys {
registerMRToEngineKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD_BYTES, TezJobConfig.TEZ_ENGINE_IFILE_READAHEAD_BYTES);
registerMRToEngineKeyTranslation(MRJobConfig.RECORDS_BEFORE_PROGRESS, TezJobConfig.RECORDS_BEFORE_PROGRESS);
-
- registerMRToEngineKeyTranslation(MRJobConfig.JOB_LOCAL_DIR, MRConfig.LOCAL_DIR);
registerMRToEngineKeyTranslation(MRJobConfig.IO_SORT_FACTOR, TezJobConfig.TEZ_ENGINE_IO_SORT_FACTOR);
@@ -149,6 +147,12 @@ public class DeprecatedKeys {
registerMRToEngineKeyTranslation(MRJobConfig.MAP_COMBINE_MIN_SPILLS, TezJobConfig.TEZ_ENGINE_COMBINE_MIN_SPILLS);
+ // Counter replacement will work in this manner, as long as TezCounters
+ // extends MRCounters and is used directly by the Mapper/Reducer.
+ // When these counters are eventually translated over to MRCounters, this
+ // may break.
+ // Framework counters, like FILESYSTEM will likely be incompatible since
+ // they enum key belongs to a different package.
registerMRToEngineKeyTranslation(MRJobConfig.COUNTERS_MAX_KEY, TezJobConfig.COUNTERS_MAX_KEY);
registerMRToEngineKeyTranslation(MRJobConfig.COUNTER_GROUP_NAME_MAX_KEY, TezJobConfig.COUNTER_GROUP_NAME_MAX_KEY);
@@ -180,7 +184,7 @@ public class DeprecatedKeys {
registerMRToEngineKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, TezJobConfig.TEZ_ENGINE_SHUFFLE_ENABLE_MEMTOMEM);
registerMRToEngineKeyTranslation(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, TezJobConfig.TEZ_ENGINE_INPUT_BUFFER_PERCENT);
-
+
registerMRToEngineKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezJobConfig.DAG_CREDENTIALS_BINARY);
registerMRToEngineKeyTranslation("map.sort.class", TezJobConfig.TEZ_ENGINE_INTERNAL_SORTER_CLASS);
@@ -188,17 +192,10 @@ public class DeprecatedKeys {
registerMRToEngineKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_ENGINE_GROUP_COMPARATOR_CLASS);
registerMRToEngineKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_ENGINE_INTERMEDIATE_INPUT_KEY_SECONDARY_COMPARATOR_CLASS);
-
- registerMRToEngineKeyTranslation(MRJobConfig.NUM_REDUCES, TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE);
- registerMRToEngineKeyTranslation(MRJobConfig.NUM_MAPS, TezJobConfig.TEZ_ENGINE_TASK_INDEGREE);
}
private static void addDeprecatedKeys() {
- // Adding deprecation for num_maps, num_reduces - in case some component in
- // Hadoop MR land uses these. Tez components already use tez-in/out-degree.
- _(MRJobConfig.NUM_REDUCES, TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE);
- _(MRJobConfig.NUM_MAPS, TezJobConfig.TEZ_ENGINE_TASK_INDEGREE);
}
private static void registerMRToEngineKeyTranslation(String mrKey,
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
index ce49726..9a6fb5b 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java
@@ -338,7 +338,6 @@ implements Processor {
throws IOException, InterruptedException {
super.localizeConfiguration(jobConf);
jobConf.setBoolean(JobContext.TASK_ISMAP, false);
- jobConf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, numMapTasks);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
index 34aaa1b..a3d5684 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/task/MRRuntimeTask.java
@@ -170,6 +170,10 @@ public class MRRuntimeTask extends RuntimeTask {
job.set(MRJobConfig.JOB_LOCAL_DIR, job.get(TezJobConfig.JOB_LOCAL_DIR));
job.set(MRConfig.LOCAL_DIR, job.get(TezJobConfig.LOCAL_DIRS));
+ if (job.get(TezJobConfig.DAG_CREDENTIALS_BINARY) != null) {
+ job.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY,
+ job.get(TezJobConfig.DAG_CREDENTIALS_BINARY));
+ }
// setup the child's attempt directories
// Do the task-type specific localization
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4c2732c9/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
index 16e783e..99aad74 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
@@ -18,28 +18,9 @@
package org.apache.tez.mapreduce.hadoop;
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.tez.common.TezJobConfig;
-import org.junit.Test;
public class TestDeprecatedKeys {
- @Test
- public void testNumTasksDeprecation() {
-
- Configuration conf = new Configuration(false);
- conf.setInt(TezJobConfig.TEZ_ENGINE_TASK_INDEGREE, 6);
- conf.setInt(TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE, 3);
-
-
- DeprecatedKeys.init();
- JobConf jobConf = new JobConf(conf);
-
- Assert.assertEquals(6, jobConf.getNumMapTasks());
- Assert.assertEquals(3, jobConf.getNumReduceTasks());
- }
-
+ // TODO Add tests
+
}
[08/50] [abbrv] git commit: TEZ-123. Cleanup package dependencies.
(bikas)
Posted by ss...@apache.org.
TEZ-123. Cleanup package dependencies. (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/8193b5fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/8193b5fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/8193b5fd
Branch: refs/heads/master
Commit: 8193b5fddbff65e9e747f08ad36bc0ad41015ab0
Parents: 8aa2f30
Author: Bikas Saha <bi...@apache.org>
Authored: Mon May 13 17:35:57 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Mon May 13 17:35:57 2013 -0700
----------------------------------------------------------------------
tez-common/pom.xml | 4 -
.../main/java/org/apache/tez/common/IDUtils.java | 10 +-
.../java/org/apache/tez/common/TezTaskContext.java | 4 +-
.../org/apache/tez/common/TezTaskReporter.java | 116 ------------
.../java/org/apache/tez/common/TezTaskStatus.java | 2 +-
.../tez/common/TezTaskUmbilicalProtocol.java | 79 ---------
.../java/org/apache/tez/dag/records/TezDAGID.java | 136 +++++++++++++++
.../java/org/apache/tez/dag/records/TezID.java | 97 ++++++++++
.../apache/tez/dag/records/TezTaskAttemptID.java | 128 ++++++++++++++
.../java/org/apache/tez/dag/records/TezTaskID.java | 126 +++++++++++++
.../org/apache/tez/dag/records/TezVertexID.java | 129 ++++++++++++++
.../org/apache/tez/dag/api/client/DAGStatus.java | 39 ----
.../apache/tez/dag/api/client/VertexStatus.java | 51 ------
tez-dag/pom.xml | 4 +
.../hadoop/mapred/MRVertexOutputCommitter.java | 6 +-
.../apache/hadoop/mapred/TezMRTypeConverter.java | 8 +-
.../org/apache/hadoop/mapred/YarnTezDagChild.java | 2 +-
.../tez/dag/api/client/impl/TezBuilderUtils.java | 8 +-
.../org/apache/tez/dag/api/impl/DAGStatus.java | 39 ++++
.../dag/api/impl/NullVertexOutputCommitter.java | 2 +-
.../org/apache/tez/dag/api/impl/VertexContext.java | 4 +-
.../tez/dag/api/impl/VertexOutputCommitter.java | 1 -
.../org/apache/tez/dag/api/impl/VertexStatus.java | 51 ++++++
.../tez/dag/api/records/TaskAttemptReport.java | 2 +-
.../org/apache/tez/dag/api/records/TaskReport.java | 4 +-
.../java/org/apache/tez/dag/app/AppContext.java | 2 +-
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 2 +-
.../apache/tez/dag/app/TaskAttemptListener.java | 2 +-
.../tez/dag/app/TaskAttemptListenerImpTezDag.java | 2 +-
.../apache/tez/dag/app/TaskHeartbeatHandler.java | 2 +-
.../main/java/org/apache/tez/dag/app/dag/DAG.java | 5 +-
.../main/java/org/apache/tez/dag/app/dag/Task.java | 4 +-
.../org/apache/tez/dag/app/dag/TaskAttempt.java | 2 +-
.../java/org/apache/tez/dag/app/dag/Vertex.java | 4 +-
.../apache/tez/dag/app/dag/VertexScheduler.java | 2 +-
.../org/apache/tez/dag/app/dag/event/DAGEvent.java | 2 +-
.../dag/app/dag/event/DAGEventCounterUpdate.java | 2 +-
.../app/dag/event/DAGEventDiagnosticsUpdate.java | 2 +-
.../dag/app/dag/event/DAGEventVertexCompleted.java | 2 +-
.../tez/dag/app/dag/event/DAGFinishEvent.java | 2 +-
.../tez/dag/app/dag/event/TaskAttemptEvent.java | 2 +-
.../event/TaskAttemptEventContainerTerminated.java | 2 +-
.../TaskAttemptEventContainerTerminating.java | 2 +-
.../event/TaskAttemptEventDiagnosticsUpdate.java | 2 +-
.../app/dag/event/TaskAttemptEventFailRequest.java | 2 +-
.../app/dag/event/TaskAttemptEventKillRequest.java | 2 +-
.../app/dag/event/TaskAttemptEventNodeFailed.java | 2 +-
.../event/TaskAttemptEventOutputConsumable.java | 2 +-
.../app/dag/event/TaskAttemptEventSchedule.java | 2 +-
.../dag/event/TaskAttemptEventStartedRemotely.java | 2 +-
.../dag/event/TaskAttemptEventStatusUpdate.java | 2 +-
.../apache/tez/dag/app/dag/event/TaskEvent.java | 2 +-
.../tez/dag/app/dag/event/TaskEventTAUpdate.java | 2 +-
.../apache/tez/dag/app/dag/event/VertexEvent.java | 2 +-
.../VertexEventSourceTaskAttemptCompleted.java | 2 +-
.../dag/event/VertexEventSourceVertexStarted.java | 2 +-
.../event/VertexEventTaskAttemptFetchFailure.java | 2 +-
.../app/dag/event/VertexEventTaskCompleted.java | 2 +-
.../app/dag/event/VertexEventTaskReschedule.java | 2 +-
.../impl/BipartiteSlowStartVertexScheduler.java | 6 +-
.../org/apache/tez/dag/app/dag/impl/DAGImpl.java | 6 +-
.../dag/impl/ImmediateStartVertexScheduler.java | 2 +-
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 4 +-
.../org/apache/tez/dag/app/dag/impl/TaskImpl.java | 6 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 12 +-
.../tez/dag/app/rm/AMSchedulerEventTAEnded.java | 2 +-
.../app/rm/AMSchedulerEventTALaunchRequest.java | 2 +-
.../tez/dag/app/rm/container/AMContainer.java | 2 +-
.../app/rm/container/AMContainerEventAssignTA.java | 2 +-
.../container/AMContainerEventLaunchRequest.java | 4 +-
.../rm/container/AMContainerEventTASucceeded.java | 2 +-
.../dag/app/rm/container/AMContainerHelpers.java | 2 +-
.../tez/dag/app/rm/container/AMContainerImpl.java | 2 +-
.../app/rm/node/AMNodeEventTaskAttemptEnded.java | 2 +-
.../rm/node/AMNodeEventTaskAttemptSucceeded.java | 2 +-
.../tez/dag/app/speculate/DefaultSpeculator.java | 6 +-
.../ExponentiallySmoothedTaskRuntimeEstimator.java | 2 +-
.../app/speculate/LegacyTaskRuntimeEstimator.java | 4 +-
.../dag/app/speculate/NullTaskRuntimesEngine.java | 4 +-
.../tez/dag/app/speculate/SpeculatorEvent.java | 6 +-
.../tez/dag/app/speculate/StartEndTimesBase.java | 6 +-
.../dag/app/speculate/TaskRuntimeEstimator.java | 4 +-
.../app/speculate/TaskSpeculationPredicate.java | 2 +-
.../tez/dag/app/taskclean/TaskCleanupEvent.java | 2 +-
.../apache/tez/dag/history/DAGHistoryEvent.java | 2 +-
.../tez/dag/history/events/DAGFinishedEvent.java | 4 +-
.../tez/dag/history/events/DAGStartedEvent.java | 2 +-
.../history/events/TaskAttemptFinishedEvent.java | 2 +-
.../history/events/TaskAttemptStartedEvent.java | 2 +-
.../tez/dag/history/events/TaskFinishedEvent.java | 2 +-
.../tez/dag/history/events/TaskStartedEvent.java | 2 +-
.../dag/history/events/VertexFinishedEvent.java | 4 +-
.../tez/dag/history/events/VertexStartedEvent.java | 2 +-
.../apache/tez/dag/utils/TezEngineChildJVM.java | 3 +-
.../tez/dag/app/dag/impl/TestTaskAttempt.java | 8 +-
.../apache/tez/dag/app/dag/impl/TestTaskImpl.java | 8 +-
tez-engine-api/pom.xml | 4 +
.../java/org/apache/tez/engine/api/Master.java | 2 +-
.../org/apache/tez/engine/records/TezDAGID.java | 136 ---------------
.../records/TezDependentTaskCompletionEvent.java | 1 +
.../java/org/apache/tez/engine/records/TezID.java | 95 ----------
.../tez/engine/records/TezTaskAttemptID.java | 127 --------------
.../org/apache/tez/engine/records/TezTaskID.java | 125 -------------
.../org/apache/tez/engine/records/TezTaskType.java | 26 ---
.../org/apache/tez/engine/records/TezVertexID.java | 129 --------------
tez-engine/pom.xml | 4 +
.../apache/tez/common/TezEngineTaskContext.java | 2 +-
.../org/apache/tez/common/TezTaskReporter.java | 116 ++++++++++++
.../tez/common/TezTaskUmbilicalProtocol.java | 79 +++++++++
.../engine/common/shuffle/impl/EventFetcher.java | 2 +-
.../tez/engine/common/shuffle/impl/Fetcher.java | 2 +-
.../engine/common/shuffle/impl/InMemoryReader.java | 2 +-
.../tez/engine/common/shuffle/impl/MapHost.java | 2 +-
.../tez/engine/common/shuffle/impl/MapOutput.java | 2 +-
.../engine/common/shuffle/impl/MergeManager.java | 4 +-
.../common/shuffle/impl/ShuffleClientMetrics.java | 2 +-
.../common/shuffle/impl/ShuffleScheduler.java | 4 +-
.../engine/common/sort/impl/ExternalSorter.java | 2 +-
.../engine/common/sort/impl/PipelinedSorter.java | 2 +-
.../common/sort/impl/dflt/DefaultSorter.java | 2 +-
.../task/local/output/TezLocalTaskOutputFiles.java | 2 +-
.../common/task/local/output/TezTaskOutput.java | 2 +-
.../task/local/output/TezTaskOutputFiles.java | 2 +-
tez-mapreduce/pom.xml | 8 +-
.../org/apache/hadoop/mapred/LocalJobRunner.java | 2 +-
.../apache/tez/mapreduce/combine/MRCombiner.java | 2 +-
.../apache/tez/mapreduce/hadoop/IDConverter.java | 8 +-
.../apache/tez/mapreduce/hadoop/MRTaskStatus.java | 2 +-
.../tez/mapreduce/hadoop/TezTypeConverters.java | 4 +-
.../mapreduce/hadoop/mapred/JobContextImpl.java | 2 +-
.../hadoop/mapred/TaskAttemptContextImpl.java | 2 +-
.../mapreduce/hadoop/mapreduce/JobContextImpl.java | 2 +-
.../org/apache/tez/mapreduce/processor/MRTask.java | 4 +-
.../mapreduce/processor/TezTaskReporterImpl.java | 2 +-
.../tez/mapreduce/TestUmbilicalProtocol.java | 2 +-
.../org/apache/tez/mapreduce/TezTestUtils.java | 8 +-
136 files changed, 1090 insertions(+), 1098 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/pom.xml
----------------------------------------------------------------------
diff --git a/tez-common/pom.xml b/tez-common/pom.xml
index eb21ea0..e1ca4c9 100644
--- a/tez-common/pom.xml
+++ b/tez-common/pom.xml
@@ -30,10 +30,6 @@
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.tez</groupId>
- <artifactId>tez-engine-api</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/IDUtils.java b/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
index a730e4a..e2cbf35 100644
--- a/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
+++ b/tez-common/src/main/java/org/apache/tez/common/IDUtils.java
@@ -20,11 +20,11 @@ package org.apache.tez.common;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
public class IDUtils {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/common/TezTaskContext.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezTaskContext.java b/tez-common/src/main/java/org/apache/tez/common/TezTaskContext.java
index f44c3d5..4eed78b 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezTaskContext.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezTaskContext.java
@@ -23,8 +23,8 @@ import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public abstract class TezTaskContext implements Writable {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/common/TezTaskReporter.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezTaskReporter.java b/tez-common/src/main/java/org/apache/tez/common/TezTaskReporter.java
deleted file mode 100644
index 478d833..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/TezTaskReporter.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.common;
-
-import java.io.IOException;
-
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.util.Progressable;
-import org.apache.tez.common.counters.TezCounter;
-import org.apache.tez.engine.api.Master;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
-
-public interface TezTaskReporter extends Progressable, Master {
-
- public void setStatus(String status);
-
- public float getProgress();
-
- public void setProgress(float progress);
-
- public void progress();
-
- public TezCounter getCounter(String group, String name);
-
- public TezCounter getCounter(Enum<?> name);
-
- public void incrCounter(String group, String counter, long amount);
-
- public void incrCounter(Enum<?> key, long amount);
-
- public void reportFatalError(TezTaskAttemptID taskAttemptId,
- Throwable exception, String logMsg);
-
- public final TezTaskReporter NULL = new TezTaskReporter() {
-
- @Override
- public TezTaskDependencyCompletionEventsUpdate getDependentTasksCompletionEvents(
- int fromEventIdx, int maxEventsToFetch,
- TezTaskAttemptID reduce) {
- return null;
- }
-
- @Override
- public void setStatus(String status) {
- }
-
- @Override
- public void setProgress(float progress) {
- }
-
- @Override
- public void progress() {
- }
-
- @Override
- public void incrCounter(Enum<?> key, long amount) {
- }
-
- @Override
- public void incrCounter(String group, String counter, long amount) {
- }
-
- @Override
- public float getProgress() {
- return 0.0f;
- }
-
- @Override
- public TezCounter getCounter(Enum<?> name) {
- return null;
- }
-
- @Override
- public TezCounter getCounter(String group, String name) {
- return null;
- }
-
- @Override
- public void reportFatalError(TezTaskAttemptID taskAttemptId,
- Throwable exception, String logMsg) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- // TODO TEZAM3
- return 0;
- }
-
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- // TODO TEZAM3
- return null;
- }
- };
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/common/TezTaskStatus.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezTaskStatus.java b/tez-common/src/main/java/org/apache/tez/common/TezTaskStatus.java
index 0b6d962..de81f87 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezTaskStatus.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezTaskStatus.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public interface TezTaskStatus extends Writable {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java b/tez-common/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
deleted file mode 100644
index 8c486aa..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.common;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.tez.common.records.ProceedToCompletionResponse;
-import org.apache.tez.engine.api.Master;
-import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-
-/** Protocol that task child process uses to contact its parent process. The
- * parent is a daemon which which polls the central master for a new map or
- * reduce task and runs it as a child process. All communication between child
- * and parent is via this protocol. */
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public interface TezTaskUmbilicalProtocol extends Master {
-
- public static final long versionID = 19L;
-
- ContainerTask getTask(ContainerContext containerContext) throws IOException;
-
- boolean statusUpdate(TezTaskAttemptID taskId, TezTaskStatus taskStatus)
- throws IOException, InterruptedException;
-
- void reportDiagnosticInfo(TezTaskAttemptID taskid, String trace) throws IOException;
-
- boolean ping(TezTaskAttemptID taskid) throws IOException;
-
- void done(TezTaskAttemptID taskid) throws IOException;
-
- void commitPending(TezTaskAttemptID taskId, TezTaskStatus taskStatus)
- throws IOException, InterruptedException;
-
- boolean canCommit(TezTaskAttemptID taskid) throws IOException;
-
- void shuffleError(TezTaskAttemptID taskId, String message) throws IOException;
-
- void fsError(TezTaskAttemptID taskId, String message) throws IOException;
-
- void fatalError(TezTaskAttemptID taskId, String message) throws IOException;
-
- // TODO TEZAM5 Can commitPending and outputReady be collapsed into a single
- // call.
- // IAC outputReady followed by commit is a little confusing - since the output
- // isn't really in place till a commit is called. Maybe rename to
- // processingComplete or some such.
-
- // TODO EVENTUALLY This is not the most useful API. Once there's some kind of
- // support for the Task handing output over to the Container, this won't rally
- // be required. i.e. InMemShuffle running as a service in the Container, or
- // the second task in getTask(). ContainerUmbilical would include getTask and
- // getServices...
-
- void outputReady(TezTaskAttemptID taskAttemptId, OutputContext outputContext)
- throws IOException;
-
- ProceedToCompletionResponse
- proceedToCompletion(TezTaskAttemptID taskAttemptId) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
new file mode 100644
index 0000000..e977dc0
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.text.NumberFormat;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+
+
+/**
+ * TezDAGID represents the immutable and unique identifier for
+ * a Tez DAG.
+ *
+ * TezDAGID consists of 2 parts. The first part is the {@link ApplicationId},
+ * that is the YARN Application ID that this DAG belongs to. The second part is
+ * the DAG number.
+ *
+ * @see ApplicationId
+ */
+public class TezDAGID extends TezID {
+
+ public static final String DAG = "dag";
+ protected static final NumberFormat idFormat = NumberFormat.getInstance();
+ static {
+ idFormat.setGroupingUsed(false);
+ idFormat.setMinimumIntegerDigits(6);
+ }
+
+ private ApplicationId applicationId;
+
+ public TezDAGID() {
+ }
+
+ /**
+ * Constructs a DAGID object from given {@link ApplicationId}.
+ * @param applicationId Application that this dag belongs to
+ * @param id the dag number
+ */
+ public TezDAGID(ApplicationId applicationId, int id) {
+ super(id);
+ if(applicationId == null) {
+ throw new IllegalArgumentException("applicationId cannot be null");
+ }
+ this.applicationId = applicationId;
+ }
+
+ /**
+ * Constructs a DAGID object from given parts.
+ * @param yarnRMIdentifier YARN RM identifier
+ * @param applicationId application number
+ * @param id the dag number
+ */
+ public TezDAGID(String yarnRMIdentifier, int appId, int id) {
+ this(BuilderUtils.newApplicationId(Long.valueOf(yarnRMIdentifier),
+ appId), id);
+ }
+
+ /** Returns the {@link ApplicationId} object that this dag belongs to */
+ public ApplicationId getApplicationId() {
+ return applicationId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!super.equals(o))
+ return false;
+
+ TezDAGID that = (TezDAGID)o;
+ return this.applicationId.equals(that.applicationId);
+ }
+
+ /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
+ @Override
+ public int compareTo(TezID o) {
+ TezDAGID that = (TezDAGID)o;
+ return this.applicationId.compareTo(that.applicationId);
+ }
+
+ @Override
+ public String toString() {
+ return appendTo(new StringBuilder(DAG)).toString();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ applicationId = BuilderUtils.newApplicationId(in.readLong(), in.readInt());
+ super.readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeLong(applicationId.getClusterTimestamp());
+ out.writeInt(applicationId.getId());
+ super.write(out);
+ }
+
+ /**
+ * Add the unique string to the given builder.
+ * @param builder the builder to append to
+ * @return the builder that was passed in
+ */
+ protected StringBuilder appendTo(StringBuilder builder) {
+ return builder.append(SEPARATOR).
+ append(applicationId.getClusterTimestamp()).
+ append(SEPARATOR).
+ append(applicationId.getId()).
+ append(SEPARATOR).
+ append(idFormat.format(id));
+ }
+
+ @Override
+ public int hashCode() {
+ return applicationId.hashCode() * 524287 + id;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java
new file mode 100644
index 0000000..7fe5d8c
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+
+/**
+ * A general identifier, which internally stores the id
+ * as an integer. This is the super class of {@link TezJobID},
+ * {@link TezTaskID} and {@link TezTaskAttemptID}.
+ *
+ * @see TezTaskID
+ * @see TezTaskAttemptID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public abstract class TezID implements WritableComparable<TezID> {
+ public static final char SEPARATOR = '_';
+ protected int id;
+
+ /** constructs an ID object from the given int */
+ public TezID(int id) {
+ this.id = id;
+ }
+
+ protected TezID() {
+ }
+
+ /** returns the int which represents the identifier */
+ public int getId() {
+ return id;
+ }
+
+ @Override
+ public String toString() {
+ return String.valueOf(id);
+ }
+
+ @Override
+ public int hashCode() {
+ return id;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if(o == null)
+ return false;
+ if (o.getClass() == this.getClass()) {
+ TezID that = (TezID) o;
+ return this.id == that.id;
+ }
+ else
+ return false;
+ }
+
+ /** Compare IDs by associated numbers */
+ public int compareTo(TezID that) {
+ return this.id - that.id;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ this.id = in.readInt();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(id);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java
new file mode 100644
index 0000000..4b2f424
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.tez.dag.records.TezTaskID;
+
+/**
+ * TezTaskAttemptID represents the immutable and unique identifier for
+ * a task attempt. Each task attempt is one particular instance of a Tez Task
+ * identified by its TezTaskID.
+ *
+ * TezTaskAttemptID consists of 2 parts. First part is the
+ * {@link TezTaskID}, that this TaskAttemptID belongs to.
+ * Second part is the task attempt number. <br>
+ * <p>
+ * Applications should never construct or parse TaskAttemptID strings
+ * , but rather use appropriate constructors or {@link #forName(String)}
+ * method.
+ *
+ * @see TezTaskID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezTaskAttemptID extends TezID {
+ public static final String ATTEMPT = "attempt";
+ private TezTaskID taskId;
+
+ public TezTaskAttemptID() {
+ taskId = new TezTaskID();
+ }
+
+ /**
+ * Constructs a TaskAttemptID object from given {@link TezTaskID}.
+ * @param taskId TaskID that this task belongs to
+ * @param id the task attempt number
+ */
+ public TezTaskAttemptID(TezTaskID taskId, int id) {
+ super(id);
+ if(taskId == null) {
+ throw new IllegalArgumentException("taskId cannot be null");
+ }
+ this.taskId = taskId;
+ }
+
+ /** Returns the {@link TezTaskID} object that this task attempt belongs to */
+ public TezTaskID getTaskID() {
+ return taskId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!super.equals(o))
+ return false;
+
+ TezTaskAttemptID that = (TezTaskAttemptID)o;
+ return this.taskId.equals(that.taskId);
+ }
+
+ /**
+ * Add the unique string to the StringBuilder
+ * @param builder the builder to append ot
+ * @return the builder that was passed in.
+ */
+ protected StringBuilder appendTo(StringBuilder builder) {
+ return taskId.appendTo(builder).append(SEPARATOR).append(id);
+ }
+
+ @Override
+ public int hashCode() {
+ return taskId.hashCode() * 539501 + id;
+ }
+
+ /**Compare TaskIds by first tipIds, then by task numbers. */
+ @Override
+ public int compareTo(TezID o) {
+ TezTaskAttemptID that = (TezTaskAttemptID)o;
+ int tipComp = this.taskId.compareTo(that.taskId);
+ if(tipComp == 0) {
+ return this.id - that.id;
+ }
+ else return tipComp;
+ }
+ @Override
+ public String toString() {
+ return appendTo(new StringBuilder(ATTEMPT)).toString();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ taskId.readFields(in);
+ super.readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ taskId.write(out);
+ super.write(out);
+ }
+
+ // FIXME TEZ DAG needs to be removed
+ public static TezTaskAttemptID read(DataInput in) throws IOException {
+ TezTaskAttemptID tId = new TezTaskAttemptID();
+ tId.readFields(in);
+ return tId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
new file mode 100644
index 0000000..d52e83b
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.tez.dag.records.TezVertexID;
+
+
+/**
+ * TaskID represents the immutable and unique identifier for
+ * a Tez Task. Each TaskID encompasses multiple attempts made to
+ * execute the Tez Task, each of which are uniquely identified by
+ * their TezTaskAttemptID.
+ *
+ * @see TezTaskAttemptID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezTaskID extends TezID {
+ public static final String TASK = "task";
+ protected static final NumberFormat idFormat = NumberFormat.getInstance();
+ static {
+ idFormat.setGroupingUsed(false);
+ idFormat.setMinimumIntegerDigits(6);
+ }
+
+ private TezVertexID vertexId;
+
+ public TezTaskID() {
+ vertexId = new TezVertexID();
+ }
+
+ /**
+ * Constructs a TaskID object from given {@link MRxApplicationID}.
+ * @param jobId JobID that this tip belongs to
+ * @param type the {@link TezTaskType} of the task
+ * @param id the tip number
+ */
+ public TezTaskID(TezVertexID vertexId, int id) {
+ super(id);
+ if(vertexId == null) {
+ throw new IllegalArgumentException("vertexId cannot be null");
+ }
+ this.vertexId = vertexId;
+ }
+
+ /** Returns the {@link TezVertexID} object that this task belongs to */
+ public TezVertexID getVertexID() {
+ return vertexId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!super.equals(o))
+ return false;
+
+ TezTaskID that = (TezTaskID)o;
+ return this.vertexId.equals(that.vertexId);
+ }
+
+ /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
+ @Override
+ public int compareTo(TezID o) {
+ TezTaskID that = (TezTaskID)o;
+ int vertexComp = this.vertexId.compareTo(that.vertexId);
+ if(vertexComp == 0) {
+ return this.id - that.id;
+ }
+ else return vertexComp;
+ }
+ @Override
+ public String toString() {
+ return appendTo(new StringBuilder(TASK)).toString();
+ }
+
+ /**
+ * Add the unique string to the given builder.
+ * @param builder the builder to append to
+ * @return the builder that was passed in
+ */
+ protected StringBuilder appendTo(StringBuilder builder) {
+ return vertexId.appendTo(builder).
+ append(SEPARATOR).
+ append(idFormat.format(id));
+ }
+
+ @Override
+ public int hashCode() {
+ return vertexId.hashCode() * 535013 + id;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ vertexId.readFields(in);
+ super.readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ vertexId.write(out);
+ super.write(out);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
new file mode 100644
index 0000000..8b2a272
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * TezVertexID represents the immutable and unique identifier for
+ * a Vertex in a Tez DAG. Each TezVertexID encompasses multiple Tez Tasks.
+ *
+ * TezVertezID consists of 2 parts. The first part is the {@link TezDAGID},
+ * that is the Tez DAG that this vertex belongs to. The second part is
+ * the vertex number.
+ *
+ * @see TezDAGID
+ * @see TezTaskID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezVertexID extends TezID {
+ public static final String VERTEX = "vertex";
+ protected static final NumberFormat idFormat = NumberFormat.getInstance();
+ static {
+ idFormat.setGroupingUsed(false);
+ idFormat.setMinimumIntegerDigits(6);
+ }
+
+ private TezDAGID dagId;
+
+ public TezVertexID() {
+ }
+
+ /**
+ * Constructs a TaskID object from given {@link TezDAGID}.
+ * @param applicationId JobID that this tip belongs to
+ * @param type the {@link TezTaskType} of the task
+ * @param id the tip number
+ */
+ public TezVertexID(TezDAGID dagId, int id) {
+ super(id);
+ if(dagId == null) {
+ throw new IllegalArgumentException("dagId cannot be null");
+ }
+ this.dagId = dagId;
+ }
+
+ /** Returns the {@link TezDAGID} object that this tip belongs to */
+ public TezDAGID getDAGId() {
+ return dagId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!super.equals(o))
+ return false;
+
+ TezVertexID that = (TezVertexID)o;
+ return this.dagId.equals(that.dagId);
+ }
+
+ /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
+ @Override
+ public int compareTo(TezID o) {
+ TezVertexID that = (TezVertexID)o;
+ return this.dagId.compareTo(that.dagId);
+ }
+
+ @Override
+ public String toString() {
+ return appendTo(new StringBuilder(VERTEX)).toString();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ dagId = new TezDAGID();
+ dagId.readFields(in);
+ super.readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ dagId.write(out);
+ super.write(out);
+ }
+
+ /**
+ * Add the unique string to the given builder.
+ * @param builder the builder to append to
+ * @return the builder that was passed in
+ */
+ protected StringBuilder appendTo(StringBuilder builder) {
+ return builder.append(SEPARATOR).
+ append(dagId.getApplicationId().getClusterTimestamp()).
+ append(SEPARATOR).
+ append(dagId.getApplicationId().getId()).
+ append(SEPARATOR).
+ append(dagId.getId()).
+ append(SEPARATOR).
+ append(idFormat.format(id));
+ }
+
+ @Override
+ public int hashCode() {
+ return dagId.hashCode() * 530017 + id;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
deleted file mode 100644
index e57b529..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.client;
-
-public class DAGStatus {
- public static enum State {
- RUNNING(1),
- SUCCEEDED(2),
- FAILED(3),
- PREP(4),
- KILLED(5);
-
- int value;
-
- State(int value) {
- this.value = value;
- }
-
- public int getValue() {
- return value;
- }
-
- };
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
deleted file mode 100644
index 34f3a5c..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.client;
-
-public class VertexStatus {
-
- public static enum State {
- RUNNING(1),
- SUCCEEDED(2),
- FAILED(3),
- PREP(4),
- KILLED(5);
-
- int value;
-
- State(int value) {
- this.value = value;
- }
-
- public int getValue() {
- return value;
- }
-
- };
-
- private State runState;
-
- public synchronized State getState() {
- return runState;
- }
-
- public synchronized void setState(State state) {
- this.runState = state;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index 8390e5f..fc03bea 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -31,6 +31,10 @@
<dependencies>
<dependency>
<groupId>org.apache.tez</groupId>
+ <artifactId>tez-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
<artifactId>tez-mapreduce</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
index bb1016a..a3b72c7 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
@@ -33,12 +33,12 @@ import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.YarnException;
-import org.apache.tez.dag.api.client.VertexStatus;
import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
import org.apache.tez.dag.api.impl.VertexContext;
import org.apache.tez.dag.api.impl.VertexOutputCommitter;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.api.impl.VertexStatus;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
public class MRVertexOutputCommitter extends VertexOutputCommitter {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/hadoop/mapred/TezMRTypeConverter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/TezMRTypeConverter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/TezMRTypeConverter.java
index 4780581..a47ef40 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/TezMRTypeConverter.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/TezMRTypeConverter.java
@@ -20,10 +20,10 @@ package org.apache.hadoop.mapred;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
public class TezMRTypeConverter {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 1854002..f140c3a 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -70,8 +70,8 @@ import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.TezEngineTaskContext;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.api.Task;
-import org.apache.tez.engine.records.TezTaskAttemptID;
import org.apache.tez.engine.runtime.RuntimeUtils;
import org.apache.tez.mapreduce.hadoop.DeprecatedKeys;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
index 1e5ffa5..5f36a58 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/impl/TezBuilderUtils.java
@@ -22,10 +22,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.tez.dag.api.records.AMInfo;
import org.apache.tez.dag.app.dag.DAGReport;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
public class TezBuilderUtils {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
new file mode 100644
index 0000000..6ede497
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/DAGStatus.java
@@ -0,0 +1,39 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.impl;
+
+public class DAGStatus {
+ public static enum State {
+ RUNNING(1),
+ SUCCEEDED(2),
+ FAILED(3),
+ PREP(4),
+ KILLED(5);
+
+ int value;
+
+ State(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return value;
+ }
+
+ };
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
index 77995a4..102bdec 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/NullVertexOutputCommitter.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.api.impl;
import java.io.IOException;
-import org.apache.tez.dag.api.client.VertexStatus.State;
+import org.apache.tez.dag.api.impl.VertexStatus.State;
public class NullVertexOutputCommitter extends VertexOutputCommitter {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
index 9789b5d..181aea5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexContext.java
@@ -20,8 +20,8 @@ package org.apache.tez.dag.api.impl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
public interface VertexContext {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
index 3024d6e..fe3e2d5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexOutputCommitter.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.tez.dag.api.client.VertexStatus;
@Public
@Unstable
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
new file mode 100644
index 0000000..9ba2ece
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/impl/VertexStatus.java
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.tez.dag.api.impl;
+
+public class VertexStatus {
+
+ public static enum State {
+ RUNNING(1),
+ SUCCEEDED(2),
+ FAILED(3),
+ PREP(4),
+ KILLED(5);
+
+ int value;
+
+ State(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return value;
+ }
+
+ };
+
+ private State runState;
+
+ public synchronized State getState() {
+ return runState;
+ }
+
+ public synchronized void setState(State state) {
+ this.runState = state;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
index 044afbb..a5a9e73 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskAttemptReport.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.api.records;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public interface TaskAttemptReport {
public abstract TezTaskAttemptID getTaskAttemptId();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
index a6605ec..a0834d7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/records/TaskReport.java
@@ -21,8 +21,8 @@ package org.apache.tez.dag.api.records;
import java.util.List;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
public interface TaskReport {
public abstract TezTaskID getTaskId();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
index df96d67..041de56 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.rm.container.AMContainerMap;
import org.apache.tez.dag.app.rm.node.AMNodeMap;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
/**
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 174d210..a0c2676 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -101,8 +101,8 @@ import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.HistoryEventHandler;
import org.apache.tez.dag.history.avro.HistoryEventType;
import org.apache.tez.dag.history.events.AMStartedEvent;
+import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.engine.common.security.JobTokenSecretManager;
-import org.apache.tez.engine.records.TezDAGID;
/**
* The Map-Reduce Application Master.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
index ef29d06..e80c8b3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListener.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.app;
import java.net.InetSocketAddress;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
/**
* This class listens for changes to the state of a Task.
*/
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index c1c88e2..d974cd8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -55,10 +55,10 @@ import org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate.TaskAttempt
import org.apache.tez.dag.app.rm.container.AMContainerImpl;
import org.apache.tez.dag.app.rm.container.AMContainerTask;
import org.apache.tez.dag.app.security.authorize.MRAMPolicyProvider;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.common.security.JobTokenSecretManager;
import org.apache.tez.engine.records.OutputContext;
import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
import org.apache.tez.engine.records.TezTaskDependencyCompletionEventsUpdate;
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
index 198ec7f..7239484 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.tez.dag.app.dag.event.TaskAttemptEvent;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventDiagnosticsUpdate;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index da0e38a..f2c3791 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -26,8 +26,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.engine.records.TezDAGID;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.api.VertexLocationHint;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
/**
* Main interface to interact with the job. Provides only getters.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
index 17f1243..6cfc2c2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
@@ -23,8 +23,8 @@ import java.util.Map;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.records.TaskReport;
import org.apache.tez.dag.api.records.TaskState;
-import org.apache.tez.engine.records.TezTaskAttemptID;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
/**
* Read only view of Task.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
index 0b8779f..d4e038a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.records.TaskAttemptReport;
import org.apache.tez.dag.api.records.TaskAttemptState;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
/**
* Read only view of TaskAttempt.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index 1e253aa..ace8a8b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -29,9 +29,9 @@ import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezTaskID;
-import org.apache.tez.engine.records.TezVertexID;
/**
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexScheduler.java
index 05a30c3..df9c28e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexScheduler.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.app.dag;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public interface VertexScheduler {
void onVertexStarted();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java
index 35cd236..1ec0222 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java
@@ -19,7 +19,7 @@
package org.apache.tez.dag.app.dag.event;
import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
/**
* This class encapsulates job related events.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java
index aba2b9b..da0724d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java
@@ -21,7 +21,7 @@ package org.apache.tez.dag.app.dag.event;
import java.util.ArrayList;
import java.util.List;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
public class DAGEventCounterUpdate extends DAGEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java
index a2ac101..bde94a0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.app.dag.event;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
public class DAGEventDiagnosticsUpdate extends DAGEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java
index 4d6cf91..80c7f42 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java
@@ -19,7 +19,7 @@
package org.apache.tez.dag.app.dag.event;
import org.apache.tez.dag.app.dag.VertexState;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezVertexID;
public class DAGEventVertexCompleted extends DAGEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGFinishEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGFinishEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGFinishEvent.java
index 15f91e3..817909f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGFinishEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGFinishEvent.java
@@ -19,7 +19,7 @@
package org.apache.tez.dag.app.dag.event;
import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezDAGID;
+import org.apache.tez.dag.records.TezDAGID;
public class DAGFinishEvent extends AbstractEvent<DAGFinishEvent.Type> {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java
index 7bbb6b7..56c03e3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java
@@ -19,7 +19,7 @@
package org.apache.tez.dag.app.dag.event;
import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
/**
* This class encapsulates task attempt related events.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java
index c2599f3..87aa313 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java
@@ -17,7 +17,7 @@
package org.apache.tez.dag.app.dag.event;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptEventContainerTerminated extends TaskAttemptEvent
implements DiagnosableEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
index bc82387..8c88cb9 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java
@@ -17,7 +17,7 @@
package org.apache.tez.dag.app.dag.event;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptEventContainerTerminating extends TaskAttemptEvent
implements DiagnosableEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventDiagnosticsUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventDiagnosticsUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventDiagnosticsUpdate.java
index 748530f..0a1ca74 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventDiagnosticsUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventDiagnosticsUpdate.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.app.dag.event;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptEventDiagnosticsUpdate extends TaskAttemptEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
index be23e17..bba8027 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventFailRequest.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.app.dag.event;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptEventFailRequest extends TaskAttemptEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java
index b0bc108..9bceb1d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java
@@ -17,7 +17,7 @@
*/
package org.apache.tez.dag.app.dag.event;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptEventKillRequest extends TaskAttemptEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
index 316f2ce..4fc718d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java
@@ -17,7 +17,7 @@
package org.apache.tez.dag.app.dag.event;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptEventNodeFailed extends TaskAttemptEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputConsumable.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputConsumable.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputConsumable.java
index ec653d4..f10209f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputConsumable.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputConsumable.java
@@ -17,8 +17,8 @@
package org.apache.tez.dag.app.dag.event;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.engine.records.OutputContext;
-import org.apache.tez.engine.records.TezTaskAttemptID;
public class TaskAttemptEventOutputConsumable extends TaskAttemptEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java
index 2f99faf..a7d4c9a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.app.dag.event;
import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java
index 27897af..5a13fc1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java
@@ -22,7 +22,7 @@ import java.util.Map;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptEventStartedRemotely extends TaskAttemptEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
index 50007fa..ef23638 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.records.TaskAttemptState;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskAttemptEventStatusUpdate extends TaskAttemptEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java
index c3c562b..c7e5faa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java
@@ -19,7 +19,7 @@
package org.apache.tez.dag.app.dag.event;
import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
/**
* this class encapsulates task related events.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java
index 66aff4b..59c7363 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.app.dag.event;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class TaskEventTAUpdate extends TaskEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java
index 4cfbfea..9e94eb5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java
@@ -19,7 +19,7 @@
package org.apache.tez.dag.app.dag.event;
import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezVertexID;
/**
* this class encapsulates vertex related events.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java
index 209bb8b..27d9a02 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java
@@ -18,8 +18,8 @@
package org.apache.tez.dag.app.dag.event;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
-import org.apache.tez.engine.records.TezVertexID;
public class VertexEventSourceTaskAttemptCompleted extends VertexEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java
index 13bd89c..cc97aa2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.app.dag.event;
-import org.apache.tez.engine.records.TezVertexID;
+import org.apache.tez.dag.records.TezVertexID;
public class VertexEventSourceVertexStarted extends VertexEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptFetchFailure.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptFetchFailure.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptFetchFailure.java
index 723338b..5b2b955 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptFetchFailure.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptFetchFailure.java
@@ -19,7 +19,7 @@ package org.apache.tez.dag.app.dag.event;
import java.util.List;
-import org.apache.tez.engine.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
public class VertexEventTaskAttemptFetchFailure extends VertexEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
index 39b0f92..79512c7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java
@@ -19,7 +19,7 @@
package org.apache.tez.dag.app.dag.event;
import org.apache.tez.dag.api.records.TaskState;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
public class VertexEventTaskCompleted extends VertexEvent {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8193b5fd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java
index 5fc09d9..9706246 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java
@@ -17,7 +17,7 @@
*/
package org.apache.tez.dag.app.dag.event;
-import org.apache.tez.engine.records.TezTaskID;
+import org.apache.tez.dag.records.TezTaskID;
public class VertexEventTaskReschedule extends VertexEvent {
private TezTaskID taskID;
[02/50] [abbrv] TEZ-25: Change DAG representation from Configuration
object to structured protobuf message
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/test/java/org/apache/tez/dag/api/TestVertexLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/api/TestVertexLocationHint.java b/tez-dag/src/test/java/org/apache/tez/dag/api/TestVertexLocationHint.java
deleted file mode 100644
index eb8c781..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/api/TestVertexLocationHint.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestVertexLocationHint {
-
- private DataInput in;
- private DataOutput out;
- private ByteArrayOutputStream bOut;
-
- @Before
- public void setup() {
- bOut = new ByteArrayOutputStream();
- out = new DataOutputStream(bOut);
- }
-
- @After
- public void teardown() {
- in = null;
- out = null;
- bOut = null;
- }
-
- @Test
- public void testNullTaskLocationHintSerDes() throws IOException {
- TaskLocationHint expected = new TaskLocationHint(null, null);
- expected.write(out);
- in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
- TaskLocationHint actual = new TaskLocationHint();
- actual.readFields(in);
- Assert.assertNull(actual.getDataLocalHosts());
- Assert.assertNull(actual.getRacks());
- }
-
- @Test
- public void testTaskLocationHintSerDes() throws IOException {
- String[] hosts = { "h1", "h2", "", null };
- String[] racks = { "r1", "r2" };
- TaskLocationHint expected = new TaskLocationHint(hosts, racks);
- expected.write(out);
- in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
- TaskLocationHint actual = new TaskLocationHint();
- actual.readFields(in);
- Assert.assertNotNull(actual.getDataLocalHosts());
- Assert.assertNotNull(actual.getRacks());
- Assert.assertArrayEquals(hosts, actual.getDataLocalHosts());
- Assert.assertArrayEquals(racks, actual.getRacks());
- }
-
- @Test
- public void testTaskLocationHintSerDes2() throws IOException {
- String[] hosts = null;
- String[] racks = { "r1", "r2" };
- TaskLocationHint expected = new TaskLocationHint(hosts, racks);
- expected.write(out);
- in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
- TaskLocationHint actual = new TaskLocationHint();
- actual.readFields(in);
- Assert.assertNull(actual.getDataLocalHosts());
- Assert.assertNotNull(actual.getRacks());
- Assert.assertArrayEquals(racks, actual.getRacks());
- }
-
- @Test
- public void testEmptyVertexLocationHintSerDes() throws IOException {
- VertexLocationHint expected = new VertexLocationHint(0);
- expected.write(out);
- in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
- VertexLocationHint actual = new VertexLocationHint();
- actual.readFields(in);
- Assert.assertEquals(0, actual.getNumTasks());
- Assert.assertNotNull(actual.getTaskLocationHints());
- Assert.assertEquals(0, actual.getTaskLocationHints().length);
- }
-
- @Test
- public void testVertexLocationHintSerDes() throws IOException {
- String[] hosts = { "h1", "h2", "", null };
- String[] racks = { "r1", "r2" };
- VertexLocationHint expected = new VertexLocationHint(4);
- expected.getTaskLocationHints()[0] = new TaskLocationHint(hosts, racks);
- expected.getTaskLocationHints()[1] = null;
- expected.getTaskLocationHints()[2] = new TaskLocationHint(null, racks);
- expected.getTaskLocationHints()[3] = new TaskLocationHint(hosts, null);
- expected.write(out);
- in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
- VertexLocationHint actual = new VertexLocationHint();
- actual.readFields(in);
-
- Assert.assertEquals(4, actual.getNumTasks());
- Assert.assertNotNull(actual.getTaskLocationHints());
- Assert.assertEquals(4, actual.getTaskLocationHints().length);
-
- Assert.assertNotNull(actual.getTaskLocationHints()[0]);
- Assert.assertNull(actual.getTaskLocationHints()[1]);
- Assert.assertNotNull(actual.getTaskLocationHints()[2]);
- Assert.assertNotNull(actual.getTaskLocationHints()[3]);
-
- Assert.assertArrayEquals(
- expected.getTaskLocationHints()[0].getDataLocalHosts(),
- actual.getTaskLocationHints()[0].getDataLocalHosts());
- Assert.assertArrayEquals(
- expected.getTaskLocationHints()[0].getRacks(),
- actual.getTaskLocationHints()[0].getRacks());
- Assert.assertNull(
- actual.getTaskLocationHints()[2].getDataLocalHosts());
- Assert.assertArrayEquals(
- expected.getTaskLocationHints()[2].getRacks(),
- actual.getTaskLocationHints()[2].getRacks());
- Assert.assertArrayEquals(
- expected.getTaskLocationHints()[3].getDataLocalHosts(),
- actual.getTaskLocationHints()[3].getDataLocalHosts());
- Assert.assertNull(
- actual.getTaskLocationHints()[3].getRacks());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 715a364..9042846 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -99,7 +99,7 @@ import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DAGConfiguration;
+import org.apache.tez.dag.api.DAGPlan.JobPlan;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
@@ -737,6 +737,14 @@ public class YARNRunner implements ClientProtocol {
List<String> vargs = new ArrayList<String>(8);
vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
+//[Debug AppMaster] Current simplest way to attach debugger to AppMaster
+// Uncomment the following, then launch a regular job, eg
+// >hadoop jar {path}\hadoop-mapreduce-examples-3.0.0-SNAPSHOT.jar pi 2 2
+// LOG.error(" !!!!!!!!!");
+// LOG.error(" !!!!!!!!! Launching AppMaster in debug/suspend mode. Attach to port 8002");
+// LOG.error(" !!!!!!!!!");
+// vargs.add("-Xdebug -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=8002,server=y,suspend=y");
+
// FIXME set up logging related properties
// TODO -Dtez.root.logger??
// MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
@@ -795,24 +803,38 @@ public class YARNRunner implements ClientProtocol {
setDAGParamsFromMRConf(dag);
- // FIXME add serialized dag conf
- DAGConfiguration dagConf = dag.serializeDag();
-
- Path dagConfFilePath = new Path(jobSubmitDir,
- TezConfiguration.DAG_AM_PLAN_CONFIG_XML);
- FSDataOutputStream dagConfOut =
- FileSystem.create(fs, dagConfFilePath,
- new FsPermission(DAG_FILE_PERMISSION));
+ // emit protobuf DAG file style
+ JobPlan dagPB = dag.createDag();
+ FSDataOutputStream dagPBOutBinaryStream = null;
+ FSDataOutputStream dagPBOutTextStream = null;
+ Path binaryPath = new Path(jobSubmitDir, TezConfiguration.DAG_AM_PLAN_PB_BINARY);
+ Path textPath = new Path(jobSubmitDir, TezConfiguration.DAG_AM_PLAN_PB_TEXT);
try {
- dagConf.writeXml(dagConfOut);
+ //binary output
+ dagPBOutBinaryStream = FileSystem.create(fs, binaryPath,
+ new FsPermission(DAG_FILE_PERMISSION));
+ dagPB.writeTo(dagPBOutBinaryStream);
+
+ // text / human-readable output
+ dagPBOutTextStream = FileSystem.create(fs, textPath,
+ new FsPermission(DAG_FILE_PERMISSION));
+ dagPBOutTextStream.writeUTF(dagPB.toString());
} finally {
- dagConfOut.close();
+ if(dagPBOutBinaryStream != null){
+ dagPBOutBinaryStream.close();
+ }
+ if(dagPBOutTextStream != null){
+ dagPBOutTextStream.close();
+ }
}
- localResources.put(TezConfiguration.DAG_AM_PLAN_CONFIG_XML,
+
+ localResources.put(TezConfiguration.DAG_AM_PLAN_PB_BINARY,
createApplicationResource(defaultFileContext,
- dagConfFilePath, LocalResourceType.FILE));
+ binaryPath, LocalResourceType.FILE));
- // FIXME add tez conf if needed
+ localResources.put(TezConfiguration.DAG_AM_PLAN_PB_TEXT,
+ createApplicationResource(defaultFileContext,
+ textPath, LocalResourceType.FILE));
// FIXME are we using MR acls for tez?
Map<ApplicationAccessType, String> acls
[12/50] [abbrv] git commit: TEZ-109. Create tests for VertexImpl.
Posted by ss...@apache.org.
TEZ-109. Create tests for VertexImpl.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/38b410e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/38b410e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/38b410e3
Branch: refs/heads/master
Commit: 38b410e3116919d31fbb9bab2ba4cc7c21cc9fee
Parents: 645c470
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu May 16 12:35:12 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu May 16 12:35:12 2013 -0700
----------------------------------------------------------------------
pom.xml | 1 +
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 70 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 950 +++++++++++++++
3 files changed, 1000 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/38b410e3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5c545c5..7fa336a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -294,6 +294,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
+ <version>2.14.1</version>
<configuration>
<forkMode>always</forkMode>
<forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/38b410e3/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 3f3ac49..aedb386 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -101,6 +101,8 @@ import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.engine.common.security.JobTokenIdentifier;
import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
+import com.google.common.annotations.VisibleForTesting;
+
/** Implementation of Vertex interface. Maintains the state machines of Vertex.
* The read and write calls use ReadWriteLock for concurrency.
@@ -112,7 +114,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private static final String LINE_SEPARATOR = System
.getProperty("line.separator");
private static final TezDependentTaskCompletionEvent[]
- EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS = new TezDependentTaskCompletionEvent[0];
+ EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS =
+ new TezDependentTaskCompletionEvent[0];
private static final Log LOG = LogFactory.getLog(VertexImpl.class);
@@ -154,7 +157,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private final List<String> diagnostics = new ArrayList<String>();
//task/attempt related datastructures
- private final Map<TezTaskID, Integer> successSourceAttemptCompletionEventNoMap =
+ @VisibleForTesting
+ final Map<TezTaskID, Integer> successSourceAttemptCompletionEventNoMap =
new HashMap<TezTaskID, Integer>();
private final Map<TezTaskAttemptID, Integer> fetchFailuresMapping =
new HashMap<TezTaskAttemptID, Integer>();
@@ -214,12 +218,14 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
.addTransition
(VertexState.RUNNING,
- EnumSet.of(VertexState.RUNNING, VertexState.SUCCEEDED, VertexState.FAILED),
+ EnumSet.of(VertexState.RUNNING, VertexState.KILLED,
+ VertexState.SUCCEEDED, VertexState.FAILED),
VertexEventType.V_TASK_COMPLETED,
new TaskCompletedTransition())
.addTransition
(VertexState.RUNNING,
- EnumSet.of(VertexState.RUNNING, VertexState.SUCCEEDED, VertexState.FAILED),
+ EnumSet.of(VertexState.RUNNING, VertexState.SUCCEEDED,
+ VertexState.FAILED),
VertexEventType.V_COMPLETED,
new VertexNoTasksCompletedTransition())
.addTransition(VertexState.RUNNING, VertexState.KILL_WAIT,
@@ -308,7 +314,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// create the topology tables
.installTopology();
- private final StateMachine<VertexState, VertexEventType, VertexEvent> stateMachine;
+ private final StateMachine<VertexState, VertexEventType, VertexEvent>
+ stateMachine;
//changing fields while the vertex is running
private int numTasks;
@@ -341,10 +348,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private VertexLocationHint vertexLocationHint;
private Map<String, LocalResource> localResources;
private Map<String, String> environment;
- private String javaOpts;
+ private final String javaOpts;
- public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, String vertexName,
- TezConfiguration conf, EventHandler eventHandler,
+ public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan,
+ String vertexName, TezConfiguration conf, EventHandler eventHandler,
TaskAttemptListener taskAttemptListener,
Token<JobTokenIdentifier> jobToken,
Credentials fsTokenCredentials, Clock clock,
@@ -573,9 +580,11 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
try {
getStateMachine().doTransition(event.getType(), event);
} catch (InvalidStateTransitonException e) {
- LOG.error("Can't handle this event at current state", e);
- addDiagnostic("Invalid event " + event.getType() +
- " on Job " + this.vertexId);
+ String message = "Invalid event " + event.getType() +
+ " on vertex " + this.vertexId +
+ " at current state " + oldState;
+ LOG.error("Can't handle " + message, e);
+ addDiagnostic(message);
eventHandler.handle(new VertexEvent(this.vertexId,
VertexEventType.INTERNAL_ERROR));
}
@@ -957,6 +966,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
@Override
public void transition(VertexImpl vertex, VertexEvent event) {
vertex.setFinishTime();
+ vertex.addDiagnostic("Vertex received Kill in NEW state.");
vertex.finished(VertexState.KILLED);
}
}
@@ -966,7 +976,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
@Override
public void transition(VertexImpl vertex, VertexEvent event) {
vertex.abortVertex(VertexStatus.State.KILLED);
- vertex.addDiagnostic("Job received Kill in INITED state.");
+ vertex.addDiagnostic("Vertex received Kill in INITED state.");
vertex.finished(VertexState.KILLED);
}
}
@@ -975,7 +985,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
implements SingleArcTransition<VertexImpl, VertexEvent> {
@Override
public void transition(VertexImpl vertex, VertexEvent event) {
- vertex.addDiagnostic("Job received Kill while in RUNNING state.");
+ vertex.addDiagnostic("Vertex received Kill while in RUNNING state.");
for (Task task : vertex.tasks.values()) {
vertex.eventHandler.handle(
new TaskEvent(task.getTaskId(), TaskEventType.T_KILL));
@@ -1133,14 +1143,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
@Override
public VertexState transition(VertexImpl vertex, VertexEvent event) {
- VertexState vertexCompleteSuccess =
- VertexImpl.checkVertexForCompletion(vertex);
- if (vertexCompleteSuccess != null) {
- return vertexCompleteSuccess;
- }
-
- // Return the current state, Job not finished yet
- return vertex.getInternalState();
+ return VertexImpl.checkVertexForCompletion(vertex);
}
}
@@ -1226,6 +1229,16 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
return appContext.getDAG();
}
+ @VisibleForTesting
+ String getProcessorName() {
+ return this.processorName;
+ }
+
+ @VisibleForTesting
+ String getJavaOpts() {
+ return this.javaOpts;
+ }
+
// TODO Eventually remove synchronization.
@Override
public synchronized List<InputSpec> getInputSpecList() {
@@ -1261,4 +1274,19 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
return outputSpecList;
}
+ @VisibleForTesting
+ VertexOutputCommitter getVertexOutputCommitter() {
+ return this.committer;
+ }
+
+ @VisibleForTesting
+ // Only to be used for testing
+ void setVertexOutputCommitter(VertexOutputCommitter committer) {
+ this.committer = committer;
+ }
+
+ @VisibleForTesting
+ VertexScheduler getVertexScheduler() {
+ return this.vertexScheduler;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/38b410e3/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
new file mode 100644
index 0000000..a1e450c
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -0,0 +1,950 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.MRVertexOutputCommitter;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.SystemClock;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.tez.dag.api.DAGProtos.DAGPlan;
+import org.apache.tez.dag.api.DAGProtos.EdgePlan;
+import org.apache.tez.dag.api.DAGProtos.PlanEdgeConnectionPattern;
+import org.apache.tez.dag.api.DAGProtos.PlanEdgeSourceType;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskConfiguration;
+import org.apache.tez.dag.api.DAGProtos.PlanTaskLocationHint;
+import org.apache.tez.dag.api.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.DAGProtos.VertexPlan;
+import org.apache.tez.dag.api.DagTypeConverters;
+import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.VertexLocationHint;
+import org.apache.tez.dag.api.impl.NullVertexOutputCommitter;
+import org.apache.tez.dag.api.impl.VertexContext;
+import org.apache.tez.dag.api.impl.VertexOutputCommitter;
+import org.apache.tez.dag.api.impl.VertexStatus.State;
+import org.apache.tez.dag.api.records.TaskState;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskHeartbeatHandler;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.VertexState;
+import org.apache.tez.dag.app.dag.event.DAGEvent;
+import org.apache.tez.dag.app.dag.event.DAGEventType;
+import org.apache.tez.dag.app.dag.event.VertexEvent;
+import org.apache.tez.dag.app.dag.event.VertexEventTaskAttemptCompleted;
+import org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted;
+import org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule;
+import org.apache.tez.dag.app.dag.event.VertexEventType;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.avro.HistoryEventType;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.engine.common.security.JobTokenIdentifier;
+import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
+import org.apache.tez.engine.records.TezDependentTaskCompletionEvent.Status;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestVertexImpl {
+
+ private static final Log LOG = LogFactory.getLog(TestVertexImpl.class);
+
+ private TezDAGID dagId;
+ private ApplicationAttemptId appAttemptId;
+ private DAGPlan dagPlan;
+ private Map<String, VertexImpl> vertices;
+ private Map<TezVertexID, VertexImpl> vertexIdMap;
+ private DrainDispatcher dispatcher;
+ private TaskAttemptListener taskAttemptListener;
+ private Credentials fsTokens;
+ private Token<JobTokenIdentifier> jobToken;
+ private Clock clock = new SystemClock();
+ private TaskHeartbeatHandler thh;
+ private AppContext appContext;
+ private VertexLocationHint vertexLocationHint;
+ private TezConfiguration conf = new TezConfiguration();
+ private Map<String, EdgeProperty> edges;
+
+ private VertexEventDispatcher vertexEventDispatcher;
+
+ private DagEventDispatcher dagEventDispatcher;
+
+ private class CountingVertexOutputCommitter extends
+ VertexOutputCommitter {
+
+ public int initCounter = 0;
+ public int setupCounter = 0;
+ public int commitCounter = 0;
+ public int abortCounter = 0;
+ private boolean throwError;
+ private boolean throwErrorOnAbort;
+
+ public CountingVertexOutputCommitter(boolean throwError,
+ boolean throwOnAbort) {
+ this.throwError = throwError;
+ this.throwErrorOnAbort = throwOnAbort;
+ }
+
+ public CountingVertexOutputCommitter() {
+ this(false, false);
+ }
+
+ @Override
+ public void init(VertexContext context) throws IOException {
+ ++initCounter;
+ }
+
+ @Override
+ public void setupVertex() throws IOException {
+ ++setupCounter;
+ }
+
+ @Override
+ public void commitVertex() throws IOException {
+ ++commitCounter;
+ if (throwError) {
+ throw new IOException("I can throwz exceptions in commit");
+ }
+ }
+
+ @Override
+ public void abortVertex(State finalState) throws IOException {
+ ++abortCounter;
+ if (throwErrorOnAbort) {
+ throw new IOException("I can throwz exceptions in abort");
+ }
+ }
+ }
+
+ private class DagEventDispatcher implements EventHandler<DAGEvent> {
+ @Override
+ public void handle(DAGEvent event) {
+ }
+ }
+
+ private class HistoryHandler implements EventHandler<DAGHistoryEvent> {
+ @Override
+ public void handle(DAGHistoryEvent event) {
+ }
+ }
+
+ private class VertexEventDispatcher
+ implements EventHandler<VertexEvent> {
+
+ @Override
+ public void handle(VertexEvent event) {
+ VertexImpl vertex = vertexIdMap.get(event.getVertexId());
+ ((EventHandler<VertexEvent>) vertex).handle(event);
+ }
+ }
+
+
+ private DAGPlan createTestDAGPlan() {
+ LOG.info("Setting up dag plan");
+ DAGPlan dag = DAGPlan.newBuilder()
+ .setName("testverteximpl")
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex1")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host1")
+ .addRack("rack1")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(0)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x1.y1")
+ .build()
+ )
+ .addOutEdgeId("e1")
+ .build()
+ )
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex2")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host2")
+ .addRack("rack2")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x2.y2")
+ .build()
+ )
+ .addOutEdgeId("e2")
+ .build()
+ )
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex3")
+ .setType(PlanVertexType.NORMAL)
+ .setProcessorName("x3.y3")
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host3")
+ .addRack("rack3")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("foo")
+ .setTaskModule("x3.y3")
+ .build()
+ )
+ .addInEdgeId("e1")
+ .addInEdgeId("e2")
+ .addOutEdgeId("e3")
+ .addOutEdgeId("e4")
+ .build()
+ )
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex4")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host4")
+ .addRack("rack4")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x4.y4")
+ .build()
+ )
+ .addInEdgeId("e3")
+ .addOutEdgeId("e5")
+ .build()
+ )
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex5")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host5")
+ .addRack("rack5")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x5.y5")
+ .build()
+ )
+ .addInEdgeId("e4")
+ .addOutEdgeId("e6")
+ .build()
+ )
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex6")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host6")
+ .addRack("rack6")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x6.y6")
+ .build()
+ )
+ .addInEdgeId("e5")
+ .addInEdgeId("e6")
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i3_v1")
+ .setInputVertexName("vertex1")
+ .setOutputClass("o1")
+ .setOutputVertexName("vertex3")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e1")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i3_v2")
+ .setInputVertexName("vertex2")
+ .setOutputClass("o2")
+ .setOutputVertexName("vertex3")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e2")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i4_v3")
+ .setInputVertexName("vertex3")
+ .setOutputClass("o3_v4")
+ .setOutputVertexName("vertex4")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e3")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i5_v3")
+ .setInputVertexName("vertex3")
+ .setOutputClass("o3_v5")
+ .setOutputVertexName("vertex5")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e4")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i6_v4")
+ .setInputVertexName("vertex4")
+ .setOutputClass("o4")
+ .setOutputVertexName("vertex6")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e5")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .addEdge(
+ EdgePlan.newBuilder()
+ .setInputClass("i6_v5")
+ .setInputVertexName("vertex5")
+ .setOutputClass("o5")
+ .setOutputVertexName("vertex6")
+ .setConnectionPattern(PlanEdgeConnectionPattern.BIPARTITE)
+ .setId("e6")
+ .setSourceType(PlanEdgeSourceType.STABLE)
+ .build()
+ )
+ .build();
+
+ return dag;
+ }
+
+ private void setupVertices() {
+ int vCnt = dagPlan.getVertexCount();
+ LOG.info("Setting up vertices from dag plan, verticesCnt=" + vCnt);
+ vertices = new HashMap<String, VertexImpl>();
+ vertexIdMap = new HashMap<TezVertexID, VertexImpl>();
+ for (int i = 0; i < vCnt; ++i) {
+ VertexPlan vPlan = dagPlan.getVertex(i);
+ TezVertexID vertexId = new TezVertexID(dagId, i+1);
+ VertexImpl v = new VertexImpl(vertexId, vPlan, vPlan.getName(), conf,
+ dispatcher.getEventHandler(), taskAttemptListener, jobToken, fsTokens,
+ clock, thh, appContext, vertexLocationHint);
+ vertices.put(vPlan.getName(), v);
+ vertexIdMap.put(vertexId, v);
+ }
+ }
+
+ private void parseVertexEdges() {
+ LOG.info("Parsing edges from dag plan, edgeCount="
+ + dagPlan.getEdgeCount());
+ int vCnt = dagPlan.getVertexCount();
+ Map<String, EdgePlan> edgePlans =
+ DagTypeConverters.createEdgePlanMapFromDAGPlan(dagPlan.getEdgeList());
+
+ for (int i = 0; i < vCnt; ++i) {
+ VertexPlan vertexPlan = dagPlan.getVertex(i);
+ Vertex vertex = vertices.get(vertexPlan.getName());
+ Map<Vertex, EdgeProperty> inVertices =
+ new HashMap<Vertex, EdgeProperty>();
+
+ Map<Vertex, EdgeProperty> outVertices =
+ new HashMap<Vertex, EdgeProperty>();
+
+ for(String inEdgeId : vertexPlan.getInEdgeIdList()){
+ EdgePlan edgePlan = edgePlans.get(inEdgeId);
+ Vertex inVertex = this.vertices.get(edgePlan.getInputVertexName());
+ EdgeProperty edgeProp = this.edges.get(inEdgeId);
+ inVertices.put(inVertex, edgeProp);
+ }
+
+ for(String outEdgeId : vertexPlan.getOutEdgeIdList()){
+ EdgePlan edgePlan = edgePlans.get(outEdgeId);
+ Vertex outVertex = this.vertices.get(edgePlan.getOutputVertexName());
+ EdgeProperty edgeProp = this.edges.get(outEdgeId);
+ outVertices.put(outVertex, edgeProp);
+ }
+ LOG.info("Setting input vertices for vertex " + vertex.getName()
+ + ", inputVerticesCnt=" + inVertices.size());
+ vertex.setInputVertices(inVertices);
+ LOG.info("Setting output vertices for vertex " + vertex.getName()
+ + ", outputVerticesCnt=" + outVertices.size());
+ vertex.setOutputVertices(outVertices);
+ }
+ }
+
+ @Before
+ public void setup() {
+ appAttemptId = BuilderUtils.newApplicationAttemptId(
+ BuilderUtils.newApplicationId(100, 1), 1);
+ dagId = new TezDAGID(appAttemptId.getApplicationId(), 1);
+ dagPlan = createTestDAGPlan();
+ dispatcher = new DrainDispatcher();
+ fsTokens = new Credentials();
+ jobToken = new Token<JobTokenIdentifier>();
+ appContext = mock(AppContext.class);
+ DAG dag = mock(DAG.class);
+ doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
+ doReturn(dag).when(appContext).getDAG();
+ doReturn(dagId).when(appContext).getDAGID();
+ doReturn(dagId).when(dag).getID();
+ setupVertices();
+ edges = DagTypeConverters.createEdgePropertyMapFromDAGPlan(
+ dagPlan.getEdgeList());
+ parseVertexEdges();
+ vertexEventDispatcher = new VertexEventDispatcher();
+ dispatcher.register(VertexEventType.class, vertexEventDispatcher);
+ dagEventDispatcher = new DagEventDispatcher();
+ dispatcher.register(DAGEventType.class, dagEventDispatcher);
+ dispatcher.register(HistoryEventType.class,
+ new HistoryHandler());
+ dispatcher.init(conf);
+ dispatcher.start();
+ }
+
+ @After
+ public void teardown() {
+ dagPlan = null;
+ this.vertices = null;
+ this.edges = null;
+ dispatcher.await();
+ dispatcher.stop();
+ }
+
+ private void initVertex(VertexImpl v) {
+ Assert.assertEquals(VertexState.NEW, v.getState());
+ v.handle(new VertexEvent(v.getVertexId(),
+ VertexEventType.V_INIT));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.INITED, v.getState());
+ }
+
+ private void startVertex(VertexImpl v) {
+ startVertex(v, true);
+ }
+
+ private void killVertex(VertexImpl v, boolean checkKillWait) {
+ v.handle(new VertexEvent(v.getVertexId(), VertexEventType.V_KILL));
+ dispatcher.await();
+ if (checkKillWait) {
+ Assert.assertEquals(VertexState.KILL_WAIT, v.getState());
+ } else {
+ Assert.assertEquals(VertexState.KILLED, v.getState());
+ }
+ }
+
+ private void startVertex(VertexImpl v,
+ boolean checkRunningState) {
+ Assert.assertEquals(VertexState.INITED, v.getState());
+ v.handle(new VertexEvent(v.getVertexId(),
+ VertexEventType.V_START));
+ dispatcher.await();
+ if (checkRunningState) {
+ Assert.assertEquals(VertexState.RUNNING, v.getState());
+ }
+ }
+
+ @Test
+ public void testVertexInit() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+
+ VertexImpl v3 = vertices.get("vertex3");
+ initVertex(v3);
+
+ Assert.assertEquals("x3.y3", v3.getProcessorName());
+ Assert.assertEquals("foo", v3.getJavaOpts());
+
+ Assert.assertEquals(2, v3.getInputSpecList().size());
+ Assert.assertEquals(2, v3.getInputVerticesCount());
+ Assert.assertEquals(2, v3.getOutputVerticesCount());
+ Assert.assertEquals(2, v3.getOutputVerticesCount());
+
+ Assert.assertTrue("vertex1".equals(v3.getInputSpecList().get(0)
+ .getVertexName())
+ || "vertex2".equals(v3.getInputSpecList().get(0)
+ .getVertexName()));
+ Assert.assertTrue("vertex1".equals(v3.getInputSpecList().get(1)
+ .getVertexName())
+ || "vertex2".equals(v3.getInputSpecList().get(1)
+ .getVertexName()));
+ Assert.assertTrue("i3_v1".equals(v3.getInputSpecList().get(0)
+ .getInputClassName())
+ || "i3_v2".equals(v3.getInputSpecList().get(0)
+ .getInputClassName()));
+ Assert.assertTrue("i3_v1".equals(v3.getInputSpecList().get(1)
+ .getInputClassName())
+ || "i3_v2".equals(v3.getInputSpecList().get(1)
+ .getInputClassName()));
+
+ Assert.assertTrue("vertex4".equals(v3.getOutputSpecList().get(0)
+ .getVertexName())
+ || "vertex5".equals(v3.getOutputSpecList().get(0)
+ .getVertexName()));
+ Assert.assertTrue("vertex4".equals(v3.getOutputSpecList().get(1)
+ .getVertexName())
+ || "vertex5".equals(v3.getOutputSpecList().get(1)
+ .getVertexName()));
+ Assert.assertTrue("o3_v4".equals(v3.getOutputSpecList().get(0)
+ .getOutputClassName())
+ || "o3_v5".equals(v3.getOutputSpecList().get(0)
+ .getOutputClassName()));
+ Assert.assertTrue("o3_v4".equals(v3.getOutputSpecList().get(1)
+ .getOutputClassName())
+ || "o3_v5".equals(v3.getOutputSpecList().get(1)
+ .getOutputClassName()));
+ }
+
+ @Test
+ public void testVertexStart() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ startVertex(v);
+ }
+
+ @Test
+ public void testBasicVertexCompletion() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ startVertex(v);
+
+ TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+ TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.RUNNING, v.getState());
+ Assert.assertEquals(1, v.getCompletedTasks());
+
+ v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+ Assert.assertEquals(2, v.getCompletedTasks());
+ }
+
+ @Test
+ @Ignore // FIXME fix verteximpl for this test to work
+ public void testDuplicateTaskCompletion() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ startVertex(v);
+
+ TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+ TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.RUNNING, v.getState());
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.RUNNING, v.getState());
+
+ v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+ }
+
+
+ @Test
+ public void testVertexFailure() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ startVertex(v);
+
+ TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.FAILED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.FAILED, v.getState());
+ String diagnostics =
+ StringUtils.join(",", v.getDiagnostics()).toLowerCase();
+ Assert.assertTrue(diagnostics.contains("task failed " + t1.toString()));
+ }
+
+ @Test
+ public void testVertexWithNoTasks() {
+ // FIXME a vertex with no tasks should not be allowed
+ VertexImpl v = vertices.get("vertex1");
+ initVertex(v);
+ startVertex(v, false);
+ dispatcher.await();
+ Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+ }
+
+ @Test
+ public void testVertexKill() {
+ VertexImpl v1 = vertices.get("vertex1");
+ killVertex(v1, false);
+ String diagnostics =
+ StringUtils.join(",", v1.getDiagnostics()).toLowerCase();
+ Assert.assertTrue(diagnostics.contains(
+ "vertex received kill in new state"));
+
+ VertexImpl v2 = vertices.get("vertex2");
+ initVertex(v2);
+ killVertex(v2, false);
+ diagnostics =
+ StringUtils.join(",", v2.getDiagnostics()).toLowerCase();
+ LOG.info("diagnostics v2: " + diagnostics);
+ Assert.assertTrue(diagnostics.contains(
+ "vertex received kill in inited state"));
+
+ VertexImpl v3 = vertices.get("vertex3");
+ initVertex(v3);
+ startVertex(v3);
+ killVertex(v3, true);
+ diagnostics =
+ StringUtils.join(",", v3.getDiagnostics()).toLowerCase();
+ Assert.assertTrue(diagnostics.contains(
+ "vertex received kill while in running state"));
+ }
+
+ @Test
+ public void testKilledTasksHandling() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ startVertex(v);
+
+ TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+ TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.KILLED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.RUNNING, v.getState());
+
+ v.handle(new VertexEventTaskCompleted(t2, TaskState.KILLED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.KILLED, v.getState());
+ }
+
+ @Test
+ public void testVertexCommitterInit() {
+ VertexImpl v2 = vertices.get("vertex2");
+ initVertex(v2);
+ Assert.assertTrue(v2.getVertexOutputCommitter()
+ instanceof NullVertexOutputCommitter);
+
+ VertexImpl v6 = vertices.get("vertex6");
+ initVertex(v6);
+ Assert.assertTrue(v6.getVertexOutputCommitter()
+ instanceof MRVertexOutputCommitter);
+ }
+
+ @Test
+ public void testVertexSchedulerInit() {
+ VertexImpl v2 = vertices.get("vertex2");
+ initVertex(v2);
+ Assert.assertTrue(v2.getVertexScheduler()
+ instanceof ImmediateStartVertexScheduler);
+
+ VertexImpl v6 = vertices.get("vertex6");
+ initVertex(v6);
+ Assert.assertTrue(v6.getVertexScheduler()
+ instanceof BipartiteSlowStartVertexScheduler);
+ }
+
+ @Test
+ public void testVertexTaskFailure() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ CountingVertexOutputCommitter committer =
+ new CountingVertexOutputCommitter();
+ v.setVertexOutputCommitter(committer);
+ startVertex(v);
+
+ TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+ TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.RUNNING, v.getState());
+
+ v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
+ v.handle(new VertexEventTaskCompleted(t2, TaskState.FAILED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.FAILED, v.getState());
+ Assert.assertEquals(0, committer.commitCounter);
+ Assert.assertEquals(1, committer.abortCounter);
+ }
+
+ @Test
+ public void testSourceVertexStartHandling() {
+ VertexImpl v4 = vertices.get("vertex4");
+ initVertex(v4);
+ VertexImpl v5 = vertices.get("vertex5");
+ initVertex(v5);
+ VertexImpl v6 = vertices.get("vertex6");
+ initVertex(v6);
+ Assert.assertEquals(VertexState.INITED, v6.getState());
+
+ startVertex(v4);
+ startVertex(v5);
+ dispatcher.await();
+ Assert.assertEquals(VertexState.RUNNING, v6.getState());
+ Assert.assertEquals(1, v6.getDistanceFromRoot());
+ }
+
+ @Test
+ public void testCounters() {
+ // FIXME need to test counters at vertex level
+ }
+
+ @Test
+ public void testDiagnostics() {
+ // FIXME need to test diagnostics in various cases
+ }
+
+ @Test
+ public void testTaskAttemptCompletionEvents() {
+ // FIXME need to test handling of task attempt events
+ }
+
+ @Test
+ public void testSourceTaskAttemptCompletionEvents() {
+ VertexImpl v4 = vertices.get("vertex4");
+ initVertex(v4);
+ VertexImpl v5 = vertices.get("vertex5");
+ initVertex(v5);
+ VertexImpl v6 = vertices.get("vertex6");
+ initVertex(v6);
+
+ startVertex(v4);
+ startVertex(v5);
+ dispatcher.await();
+ Assert.assertEquals(VertexState.RUNNING, v6.getState());
+
+ TezTaskID t1_v4 = new TezTaskID(v4.getVertexId(), 0);
+ TezTaskID t2_v4 = new TezTaskID(v4.getVertexId(), 1);
+ TezTaskID t1_v5 = new TezTaskID(v5.getVertexId(), 0);
+ TezTaskID t2_v5 = new TezTaskID(v5.getVertexId(), 1);
+
+ TezTaskAttemptID ta1_t1_v4 = new TezTaskAttemptID(t1_v4, 0);
+ TezTaskAttemptID ta2_t1_v4 = new TezTaskAttemptID(t1_v4, 0);
+ TezTaskAttemptID ta1_t2_v4 = new TezTaskAttemptID(t2_v4, 0);
+ TezTaskAttemptID ta1_t1_v5 = new TezTaskAttemptID(t1_v5, 0);
+ TezTaskAttemptID ta1_t2_v5 = new TezTaskAttemptID(t2_v5, 0);
+ TezTaskAttemptID ta2_t2_v5 = new TezTaskAttemptID(t2_v5, 0);
+
+ TezDependentTaskCompletionEvent cEvt1 =
+ new TezDependentTaskCompletionEvent(1, ta1_t1_v4,
+ Status.FAILED, "", 3);
+ TezDependentTaskCompletionEvent cEvt2 =
+ new TezDependentTaskCompletionEvent(2, ta2_t1_v4,
+ Status.SUCCEEDED, "", 4);
+ TezDependentTaskCompletionEvent cEvt3 =
+ new TezDependentTaskCompletionEvent(2, ta1_t2_v4,
+ Status.SUCCEEDED, "", 5);
+ TezDependentTaskCompletionEvent cEvt4 =
+ new TezDependentTaskCompletionEvent(2, ta1_t1_v5,
+ Status.SUCCEEDED, "", 5);
+ TezDependentTaskCompletionEvent cEvt5 =
+ new TezDependentTaskCompletionEvent(1, ta1_t2_v5,
+ Status.FAILED, "", 3);
+ TezDependentTaskCompletionEvent cEvt6 =
+ new TezDependentTaskCompletionEvent(2, ta2_t2_v5,
+ Status.SUCCEEDED, "", 4);
+
+ v4.handle(new VertexEventTaskAttemptCompleted(cEvt1));
+ v4.handle(new VertexEventTaskAttemptCompleted(cEvt2));
+ v4.handle(new VertexEventTaskAttemptCompleted(cEvt3));
+ v5.handle(new VertexEventTaskAttemptCompleted(cEvt4));
+ v5.handle(new VertexEventTaskAttemptCompleted(cEvt5));
+ v5.handle(new VertexEventTaskAttemptCompleted(cEvt6));
+
+ v4.handle(new VertexEventTaskCompleted(t1_v4, TaskState.SUCCEEDED));
+ v4.handle(new VertexEventTaskCompleted(t2_v4, TaskState.SUCCEEDED));
+ v5.handle(new VertexEventTaskCompleted(t1_v5, TaskState.SUCCEEDED));
+ v5.handle(new VertexEventTaskCompleted(t2_v5, TaskState.SUCCEEDED));
+ dispatcher.await();
+
+ Assert.assertEquals(VertexState.SUCCEEDED, v4.getState());
+ Assert.assertEquals(VertexState.SUCCEEDED, v5.getState());
+
+ Assert.assertEquals(VertexState.RUNNING, v6.getState());
+ Assert.assertEquals(4, v6.successSourceAttemptCompletionEventNoMap.size());
+ Assert.assertEquals(6, v6.getTaskAttemptCompletionEvents(0, 100).length);
+ }
+
+ @Test
+ public void testDAGEventGeneration() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ startVertex(v);
+
+ TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+ TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+
+ }
+
+ @Test
+ public void testTaskReschedule() {
+ // For downstream failures
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ CountingVertexOutputCommitter committer =
+ new CountingVertexOutputCommitter();
+ v.setVertexOutputCommitter(committer);
+
+ startVertex(v);
+
+ TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+ TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ v.handle(new VertexEventTaskReschedule(t1));
+ // FIXME need to handle dups
+ // v.handle(new VertexEventTaskReschedule(t1));
+ v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.RUNNING, v.getState());
+ Assert.assertEquals(0, committer.commitCounter);
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+ Assert.assertEquals(1, committer.commitCounter);
+
+ }
+
+ @Test
+ public void testVertexCommit() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ CountingVertexOutputCommitter committer =
+ new CountingVertexOutputCommitter();
+ v.setVertexOutputCommitter(committer);
+
+ startVertex(v);
+
+ TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+ TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+ Assert.assertEquals(1, committer.commitCounter);
+
+ v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
+ Assert.assertEquals(1, committer.commitCounter);
+ Assert.assertEquals(0, committer.abortCounter);
+ Assert.assertEquals(0, committer.initCounter); // already done in init
+ Assert.assertEquals(0, committer.setupCounter); // already done in init
+ }
+
+ @Test
+ public void testCommitterInitAndSetup() {
+ // FIXME need to add a test for this
+ }
+
+ @Test
+ public void testTaskAttemptFetchFailureHandling() {
+ // FIXME needs testing
+ }
+
+ @Test
+ public void testBadCommitter() {
+ VertexImpl v = vertices.get("vertex2");
+ initVertex(v);
+ CountingVertexOutputCommitter committer =
+ new CountingVertexOutputCommitter(true, true);
+ v.setVertexOutputCommitter(committer);
+
+ startVertex(v);
+
+ TezTaskID t1 = new TezTaskID(v.getVertexId(), 0);
+ TezTaskID t2 = new TezTaskID(v.getVertexId(), 1);
+
+ v.handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
+ v.handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
+ dispatcher.await();
+ Assert.assertEquals(VertexState.FAILED, v.getState());
+ Assert.assertEquals(1, committer.commitCounter);
+
+ // FIXME need to verify whether abort needs to be called if commit fails
+ Assert.assertEquals(0, committer.abortCounter);
+ Assert.assertEquals(0, committer.initCounter); // already done in init
+ Assert.assertEquals(0, committer.setupCounter); // already done in init
+ }
+
+}
[44/50] [abbrv] git commit: Fix build after YARN-717,
YARN-720. (sseth)
Posted by ss...@apache.org.
Fix build after YARN-717, YARN-720. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/86523292
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/86523292
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/86523292
Branch: refs/heads/master
Commit: 8652329273978266490806028a4c274af33d213e
Parents: eeaaf23
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jun 3 09:25:46 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jun 3 09:25:46 2013 -0700
----------------------------------------------------------------------
.../dag/app/launcher/ContainerLauncherImpl.java | 13 ++++++-------
.../apache/tez/dag/app/rm/NMCommunicatorEvent.java | 8 ++++----
.../dag/app/rm/NMCommunicatorStopRequestEvent.java | 4 ++--
.../java/org/apache/tez/mapreduce/YARNRunner.java | 7 ++-----
4 files changed, 14 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index 5b23c75..b71af02 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.YarnException;
@@ -46,7 +45,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.service.AbstractService;
@@ -123,10 +122,10 @@ public class ContainerLauncherImpl extends AbstractService implements
// store enough information to be able to cleanup the container
private ContainerId containerID;
final private String containerMgrAddress;
- private ContainerToken containerToken;
+ private Token containerToken;
public Container(ContainerId containerID,
- String containerMgrAddress, ContainerToken containerToken) {
+ String containerMgrAddress, Token containerToken) {
this.state = ContainerState.PREP;
this.containerMgrAddress = containerMgrAddress;
this.containerID = containerID;
@@ -355,7 +354,7 @@ public class ContainerLauncherImpl extends AbstractService implements
}
protected ContainerManager getCMProxy(ContainerId containerID,
- final String containerManagerBindAddr, ContainerToken containerToken)
+ final String containerManagerBindAddr, Token containerToken)
throws IOException {
final InetSocketAddress cmAddr =
@@ -364,8 +363,8 @@ public class ContainerLauncherImpl extends AbstractService implements
// the user in createRemoteUser in this context has to be ContainerID
UserGroupInformation user = UserGroupInformation
.createRemoteUser(containerID.toString());
- Token<ContainerTokenIdentifier> token = ProtoUtils.convertFromProtoFormat(
- containerToken, cmAddr);
+ org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token = ProtoUtils
+ .convertFromProtoFormat(containerToken, cmAddr);
user.addToken(token);
ContainerManager proxy = user
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
index 38f54c6..8bdeb28 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
@@ -19,18 +19,18 @@
package org.apache.tez.dag.app.rm;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.event.AbstractEvent;
public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType> {
private final ContainerId containerId;
private final NodeId nodeId;
- private final ContainerToken containerToken;
+ private final Token containerToken;
public NMCommunicatorEvent(ContainerId containerId, NodeId nodeId,
- ContainerToken containerToken, NMCommunicatorEventType type) {
+ Token containerToken, NMCommunicatorEventType type) {
super(type);
this.containerId = containerId;
this.nodeId = nodeId;
@@ -45,7 +45,7 @@ public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType>
return this.nodeId;
}
- public ContainerToken getContainerToken() {
+ public Token getContainerToken() {
return this.containerToken;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
index 8f09a48..277d1e7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
@@ -19,13 +19,13 @@
package org.apache.tez.dag.app.rm;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Token;
public class NMCommunicatorStopRequestEvent extends NMCommunicatorEvent {
public NMCommunicatorStopRequestEvent(ContainerId containerId, NodeId nodeId,
- ContainerToken containerToken) {
+ Token containerToken) {
super(containerId, nodeId, containerToken,
NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 801ce8a..572437b 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -70,7 +70,6 @@ import org.apache.hadoop.mapreduce.v2.LogParams;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.YarnException;
@@ -1141,13 +1140,11 @@ public class YARNRunner implements ClientProtocol {
private static void addLog4jSystemProperties(String logLevel,
List<String> vargs) {
vargs.add("-Dlog4j.configuration=container-log4j.properties");
- // TODO Fix the remaining properties after YARN-720 is fixed.
- // May need to introduce a log4j properties file for tez tasks.
- vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "="
+ vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
// Setting this to 0 to avoid log size restrictions.
// Should be enforced by YARN.
- vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + 0);
+ vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=" + 0);
vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
}
[27/50] [abbrv] git commit: TEZ-155. Basic fixes for Hive (gunther
via bikas)
Posted by ss...@apache.org.
TEZ-155. Basic fixes for Hive (gunther via bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/5d8adeb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/5d8adeb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/5d8adeb5
Branch: refs/heads/master
Commit: 5d8adeb52d1ef46b4ea3874a675ace930c45bc91
Parents: 4c2732c
Author: Bikas Saha <bi...@apache.org>
Authored: Thu May 23 18:34:03 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Thu May 23 18:34:03 2013 -0700
----------------------------------------------------------------------
.../tez/mapreduce/ClientServiceDelegate.java | 21 ++++++++++-----
.../org/apache/tez/mapreduce/DAGJobStatus.java | 11 ++++---
2 files changed, 20 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5d8adeb5/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
index 44ccc07..1335a44 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ClientServiceDelegate.java
@@ -24,7 +24,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.v2.LogParams;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.mapreduce.TaskCompletionEvent;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -87,12 +89,17 @@ public class ClientServiceDelegate {
}
public JobStatus getJobStatus(JobID oldJobID) throws IOException {
+ org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
+ TypeConverter.toYarn(oldJobID);
+ String user = UserGroupInformation.getCurrentUser().getShortUserName();
+ String jobFile = MRApps.getJobFile(conf, user, oldJobID);
+
try {
if(dagClient == null) {
appReport = getAppReport(oldJobID);
if(appReport.getYarnApplicationState() != YarnApplicationState.RUNNING) {
// if job not running return status from appReport;
- return getJobStatusFromRM(appReport);
+ return getJobStatusFromRM(appReport, jobFile);
} else {
// job is running. create dag am client
dagClient = tezClient.getDAGClient(appReport.getHost(),
@@ -102,7 +109,7 @@ public class ClientServiceDelegate {
}
// return status from client. use saved appReport for queue etc
DAGStatus dagStatus = dagClient.getDAGStatus(currentDAGId);
- return new DAGJobStatus(appReport, dagStatus);
+ return new DAGJobStatus(appReport, dagStatus, jobFile);
} catch (TezRemoteException e) {
// AM not responding
dagClient = null;
@@ -116,20 +123,20 @@ public class ClientServiceDelegate {
}
// final fallback
- return getJobStatusFromRM(appReport);
+ return getJobStatusFromRM(appReport, jobFile);
}
- private JobStatus getJobStatusFromRM(ApplicationReport appReport) {
+ private JobStatus getJobStatusFromRM(ApplicationReport appReport, String jobFile) {
JobStatus jobStatus =
- new DAGJobStatus(appReport, null);
+ new DAGJobStatus(appReport, null, jobFile);
return jobStatus;
}
public org.apache.hadoop.mapreduce.TaskReport[] getTaskReports(
JobID oldJobID, TaskType taskType)
throws IOException{
- // FIXME need support to query task reports?
- throw new UnsupportedOperationException();
+ // TEZ-146: need to return real task reports
+ return new org.apache.hadoop.mapreduce.TaskReport[0];
}
public boolean killTask(TaskAttemptID taskAttemptID, boolean fail)
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5d8adeb5/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
index 08c61f8..10d05b3 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/DAGJobStatus.java
@@ -39,13 +39,15 @@ import org.mortbay.log.Log;
public class DAGJobStatus extends JobStatus {
+ String jobFile;
private final ApplicationReport report;
private final DAGStatus dagStatus;
- public DAGJobStatus(ApplicationReport appReport, DAGStatus dagStatus) {
+ public DAGJobStatus(ApplicationReport appReport, DAGStatus dagStatus, String jobFile) {
super();
this.report = appReport;
this.dagStatus = dagStatus;
+ this.jobFile = jobFile;
}
@Override
@@ -215,8 +217,8 @@ public class DAGJobStatus extends JobStatus {
@Override
public synchronized JobPriority getPriority() {
- // TODO Auto-generated method stub
- return super.getPriority();
+ // TEX-147: return real priority
+ return JobPriority.NORMAL;
}
@Override
@@ -251,8 +253,7 @@ public class DAGJobStatus extends JobStatus {
@Override
public String getJobFile() {
- // FIXME
- throw new UnsupportedOperationException();
+ return jobFile;
}
@Override
[47/50] [abbrv] git commit: TEZ-175. Cleanup logs.
Posted by ss...@apache.org.
TEZ-175. Cleanup logs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/9feab053
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/9feab053
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/9feab053
Branch: refs/heads/master
Commit: 9feab053fc0508f9ec6b791db8afdd4eebc11113
Parents: 9f040cf
Author: Hitesh Shah <hi...@apache.org>
Authored: Mon Jun 3 16:42:28 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Mon Jun 3 16:46:40 2013 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/mapred/YarnTezDagChild.java | 31 ++--
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 102 +++++-----
.../tez/dag/app/TaskAttemptListenerImpTezDag.java | 60 +++---
.../org/apache/tez/dag/app/dag/impl/DAGImpl.java | 60 +++---
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 104 +++++-----
.../org/apache/tez/dag/app/dag/impl/TaskImpl.java | 164 ++++++++-------
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 36 ++--
.../tez/dag/app/rm/container/AMContainerImpl.java | 89 +++++----
.../org/apache/tez/dag/app/rm/node/AMNodeImpl.java | 24 ++-
.../engine/common/shuffle/impl/InMemoryWriter.java | 46 ++--
.../engine/common/sort/impl/ExternalSorter.java | 38 ++--
.../engine/common/sort/impl/IFileOutputStream.java | 14 +-
.../common/sort/impl/dflt/DefaultSorter.java | 88 ++++----
.../task/local/output/TezTaskOutputFiles.java | 38 ++--
.../engine/lib/output/LocalOnFileSorterOutput.java | 12 +-
.../apache/tez/engine/runtime/RuntimeUtils.java | 29 ++-
.../org/apache/hadoop/mapred/LocalJobRunner.java | 122 ++++++------
.../mapreduce/split/SplitMetaInfoReaderTez.java | 12 +-
.../apache/tez/mapreduce/combine/MRCombiner.java | 7 +-
.../hadoop/MultiStageMRConfToTezTranslator.java | 34 ++--
.../org/apache/tez/mapreduce/processor/MRTask.java | 147 +++++++-------
.../java/org/apache/tez/mapreduce/YARNRunner.java | 64 ++++---
22 files changed, 686 insertions(+), 635 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 6fd6eff..33e3972 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -146,9 +146,10 @@ public class YarnTezDagChild {
}
taskContext = (TezEngineTaskContext) containerTask
.getTezEngineTaskContext();
- LOG.info("DEBUG: New container task context:"
- + taskContext.toString());
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("New container task context:"
+ + taskContext.toString());
+ }
taskAttemptId = taskContext.getTaskAttemptId();
final Task t = createAndConfigureTezTask(taskContext, umbilical,
@@ -200,25 +201,25 @@ public class YarnTezDagChild {
/**
* Configure mapred-local dirs. This config is used by the task for finding
* out an output directory.
- * @throws IOException
+ * @throws IOException
*/
/**
* Configure tez-local-dirs, tez-localized-file-dir, etc. Also create these
* dirs.
*/
-
+
private static void configureLocalDirs(Configuration conf) throws IOException {
String[] localSysDirs = StringUtils.getTrimmedStrings(
System.getenv(Environment.LOCAL_DIRS.name()));
conf.setStrings(TezJobConfig.LOCAL_DIRS, localSysDirs);
conf.set(TezJobConfig.TASK_LOCAL_RESOURCE_DIR,
System.getenv(Environment.PWD.name()));
-
+
LOG.info(TezJobConfig.LOCAL_DIRS + " for child: " +
conf.get(TezJobConfig.LOCAL_DIRS));
LOG.info(TezJobConfig.TASK_LOCAL_RESOURCE_DIR + " for child: "
+ conf.get(TezJobConfig.TASK_LOCAL_RESOURCE_DIR));
-
+
LocalDirAllocator lDirAlloc = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
Path workDir = null;
// First, try to find the JOB_LOCAL_DIR on this host.
@@ -250,7 +251,7 @@ public class YarnTezDagChild {
}
conf.set(TezJobConfig.JOB_LOCAL_DIR, workDir.toString());
}
-
+
private static Task createAndConfigureTezTask(
TezEngineTaskContext taskContext, TezTaskUmbilicalProtocol master,
Credentials cxredentials, Token<JobTokenIdentifier> jobToken,
@@ -260,10 +261,10 @@ public class YarnTezDagChild {
// set tcp nodelay
conf.setBoolean("ipc.client.tcpnodelay", true);
conf.setInt(TezJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId);
-
+
configureLocalDirs(conf);
-
-
+
+
// FIXME need Input/Output vertices else we have this hack
if (taskContext.getInputSpecList().isEmpty()) {
taskContext.getInputSpecList().add(
@@ -281,18 +282,18 @@ public class YarnTezDagChild {
// and processor then inits inputs and outputs
return t;
}
-
+
private static void runTezTask(
- Task t, TezTaskUmbilicalProtocol master, Configuration conf)
+ Task t, TezTaskUmbilicalProtocol master, Configuration conf)
throws IOException, InterruptedException {
// use job-specified working directory
FileSystem.get(conf).setWorkingDirectory(getWorkingDirectory(conf));
-
+
// Run!
t.run();
t.close();
}
-
+
private static Path getWorkingDirectory(Configuration conf) {
String name = conf.get(JobContext.WORKING_DIR);
if (name != null) {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 821dd71..a4ca97e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -128,7 +128,7 @@ import org.apache.tez.engine.common.security.JobTokenSecretManager;
@SuppressWarnings("rawtypes")
public class DAGAppMaster extends CompositeService {
-
+
private static final Log LOG = LogFactory.getLog(DAGAppMaster.class);
/**
@@ -137,7 +137,7 @@ public class DAGAppMaster extends CompositeService {
public static final int SHUTDOWN_HOOK_PRIORITY = 30;
private Clock clock;
- private final DAGPlan jobPlan;
+ private final DAGPlan dagPlan;
private long dagsStartTime;
private final long startTime;
private final long appSubmitTime;
@@ -154,7 +154,7 @@ public class DAGAppMaster extends CompositeService {
// TODO Recovery
//private Map<TezTaskID, TaskInfo> completedTasksFromPreviousRun;
private AppContext context;
- private TezConfiguration conf;
+ private TezConfiguration conf;
private Dispatcher dispatcher;
// TODO Recovery
//private Recovery recoveryServ;
@@ -176,7 +176,7 @@ public class DAGAppMaster extends CompositeService {
private HistoryEventHandler historyEventHandler;
private DAGAppMasterState state;
-
+
DAGClientServer clientRpcServer;
private DAGClientHandler clientHandler;
@@ -195,7 +195,7 @@ public class DAGAppMaster extends CompositeService {
ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
Clock clock, long appSubmitTime, DAGPlan dagPB) {
super(DAGAppMaster.class.getName());
- this.jobPlan = dagPB;
+ this.dagPlan = dagPB;
this.clock = clock;
this.startTime = clock.getTime();
this.appSubmitTime = appSubmitTime;
@@ -214,9 +214,9 @@ public class DAGAppMaster extends CompositeService {
public void init(final Configuration tezConf) {
this.state = DAGAppMasterState.INITED;
-
+
assert tezConf instanceof TezConfiguration;
-
+
this.conf = (TezConfiguration) tezConf;
conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
@@ -226,10 +226,10 @@ public class DAGAppMaster extends CompositeService {
// Job name is the same as the app name util we support DAG of jobs
// for an app later
- appName = jobPlan.getName();
+ appName = dagPlan.getName();
dagId = new TezDAGID(appAttemptID.getApplicationId(), 1);
-
+
clientHandler = new DAGClientHandler();
// TODO Committer.
@@ -237,7 +237,7 @@ public class DAGAppMaster extends CompositeService {
dispatcher = createDispatcher();
addIfService(dispatcher);
-
+
clientRpcServer = new DAGClientServer(clientHandler);
addIfService(clientRpcServer);
@@ -432,7 +432,7 @@ public class DAGAppMaster extends CompositeService {
LOG.warn("No handler for event type: " + event.getType());
}
}
-
+
private class DAGAppMasterEventHandler implements
EventHandler<DAGAppMasterEvent> {
@Override
@@ -440,7 +440,7 @@ public class DAGAppMaster extends CompositeService {
DAGAppMaster.this.handle(event);
}
}
-
+
private class DAGFinishEventHandler implements EventHandler<DAGFinishEvent> {
@Override
public void handle(DAGFinishEvent event) {
@@ -466,9 +466,9 @@ public class DAGAppMaster extends CompositeService {
*/
// TODO:currently just wait for some time so clients can know the
// final states. Will be removed once RM come on.
-
+
setStateOnDAGCompletion();
-
+
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
@@ -660,19 +660,19 @@ public class DAGAppMaster extends CompositeService {
public TaskAttemptListener getTaskAttemptListener() {
return taskAttemptListener;
}
-
+
public ContainerId getAppContainerId() {
return containerID;
}
-
+
public String getAppNMHost() {
return nmHost;
}
-
+
public int getAppNMPort() {
return nmPort;
}
-
+
public int getAppNMHttpPort() {
return nmHttpPort;
}
@@ -680,15 +680,15 @@ public class DAGAppMaster extends CompositeService {
public DAGAppMasterState getState() {
return state;
}
-
+
public List<String> getDiagnostics() {
return dag.getDiagnostics();
}
-
+
public float getProgress() {
return dag.getProgress();
}
-
+
void setStateOnDAGCompletion() {
DAGAppMasterState oldState = state;
if(state == DAGAppMasterState.RUNNING) {
@@ -709,7 +709,7 @@ public class DAGAppMaster extends CompositeService {
}
LOG.info("On DAG completion. Old state: " + oldState + " new state: " + state);
}
-
+
class DAGClientHandler implements DAGClient {
@Override
@@ -718,22 +718,22 @@ public class DAGAppMaster extends CompositeService {
}
@Override
- public DAGStatus getDAGStatus(String dagIdStr)
+ public DAGStatus getDAGStatus(String dagIdStr)
throws IOException, TezRemoteException {
return getDAG(dagIdStr).getDAGStatus();
}
@Override
- public VertexStatus getVertexStatus(String dagIdStr, String vertexName)
+ public VertexStatus getVertexStatus(String dagIdStr, String vertexName)
throws IOException, TezRemoteException{
VertexStatus status = getDAG(dagIdStr).getVertexStatus(vertexName);
if(status == null) {
throw new TezRemoteException("Unknown vertexName: " + vertexName);
}
-
+
return status;
}
-
+
DAG getDAG(String dagIdStr) throws IOException, TezRemoteException {
TezDAGID dagId = TezDAGID.fromString(dagIdStr);
if(dagId == null) {
@@ -759,11 +759,11 @@ public class DAGAppMaster extends CompositeService {
this.conf = config;
}
- @Override
+ @Override
public DAGAppMaster getAppMaster() {
return DAGAppMaster.this;
}
-
+
@Override
public ApplicationAttemptId getApplicationAttemptId() {
return appAttemptID;
@@ -859,7 +859,7 @@ public class DAGAppMaster extends CompositeService {
public void start() {
this.state = DAGAppMasterState.RUNNING;
-
+
// TODO Recovery
// Pull completedTasks etc from recovery
/*
@@ -868,9 +868,9 @@ public class DAGAppMaster extends CompositeService {
amInfos = recoveryServ.getAMInfos();
}
*/
-
+
// /////////////////// Create the job itself.
- dag = createDAG(jobPlan);
+ dag = createDAG(dagPlan);
// End of creating the job.
@@ -893,7 +893,7 @@ public class DAGAppMaster extends CompositeService {
startTime, dagsStartTime, appSubmitTime);
dispatcher.getEventHandler().handle(
new DAGHistoryEvent(this.dagId, startEvent));
-
+
// All components have started, start the job.
startDags();
}
@@ -922,7 +922,7 @@ public class DAGAppMaster extends CompositeService {
((EventHandler<DAGEvent>)context.getDAG()).handle(event);
}
}
-
+
private class TaskEventDispatcher implements EventHandler<TaskEvent> {
@SuppressWarnings("unchecked")
@Override
@@ -959,7 +959,7 @@ public class DAGAppMaster extends CompositeService {
((EventHandler<VertexEvent>) vertex).handle(event);
}
}
-
+
private static void validateInputParam(String value, String param)
throws IOException {
if (value == null) {
@@ -998,22 +998,22 @@ public class DAGAppMaster extends CompositeService {
// TODO change this once the client is ready.
String type;
TezConfiguration conf = new TezConfiguration(new YarnConfiguration());
-
- DAGPlan jobPlan = null;
+
+ DAGPlan dagPlan = null;
if (cliParser.hasOption(OPT_PREDEFINED)) {
LOG.info("Running with PreDefined configuration");
type = cliParser.getOptionValue(OPT_PREDEFINED, "mr");
LOG.info("Running job type: " + type);
if (type.equals("mr")) {
- jobPlan = MRRExampleHelper.createDAGConfigurationForMR();
+ dagPlan = MRRExampleHelper.createDAGConfigurationForMR();
} else if (type.equals("mrr")) {
- jobPlan = MRRExampleHelper.createDAGConfigurationForMRR();
+ dagPlan = MRRExampleHelper.createDAGConfigurationForMRR();
}
- }
+ }
else {
// Read the protobuf DAG
- DAGPlan.Builder dagPlanBuilder = DAGPlan.newBuilder();
+ DAGPlan.Builder dagPlanBuilder = DAGPlan.newBuilder();
FileInputStream dagPBBinaryStream = null;
try {
dagPBBinaryStream = new FileInputStream(TezConfiguration.DAG_AM_PLAN_PB_BINARY);
@@ -1021,17 +1021,19 @@ public class DAGAppMaster extends CompositeService {
}
finally {
if(dagPBBinaryStream != null){
- dagPBBinaryStream.close();
+ dagPBBinaryStream.close();
}
}
- jobPlan = dagPlanBuilder.build();
+ dagPlan = dagPlanBuilder.build();
}
- LOG.info("XXXX Running a DAG with "
- + jobPlan.getVertexCount() + " vertices ");
- for (VertexPlan v : jobPlan.getVertexList()) {
- LOG.info("XXXX DAG has vertex " + v.getName());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Running a DAG with "
+ + dagPlan.getVertexCount() + " vertices ");
+ for (VertexPlan v : dagPlan.getVertexList()) {
+ LOG.debug("DAG has vertex " + v.getName());
+ }
}
String jobUserName = System
@@ -1041,8 +1043,8 @@ public class DAGAppMaster extends CompositeService {
// SIGTERM I have a chance to write out the job history. I'll be closing
// the objects myself.
conf.setBoolean("fs.automatic.close", false);
-
- Map<String, String> config = DagTypeConverters.createSettingsMapFromDAGPlan(jobPlan.getJobSettingList());
+
+ Map<String, String> config = DagTypeConverters.createSettingsMapFromDAGPlan(dagPlan.getJobSettingList());
for(Entry<String, String> entry : config.entrySet()) {
conf.set(entry.getKey(), entry.getValue());
}
@@ -1050,7 +1052,7 @@ public class DAGAppMaster extends CompositeService {
DAGAppMaster appMaster =
new DAGAppMaster(applicationAttemptId, containerId, nodeHostString,
Integer.parseInt(nodePortString),
- Integer.parseInt(nodeHttpPortString), appSubmitTime, jobPlan);
+ Integer.parseInt(nodeHttpPortString), appSubmitTime, dagPlan);
ShutdownHookManager.get().addShutdownHook(
new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
@@ -1117,7 +1119,7 @@ public class DAGAppMaster extends CompositeService {
}
});
}
-
+
@SuppressWarnings("unchecked")
private void sendEvent(Event<?> event) {
dispatcher.getEventHandler().handle(event);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
index 7600db4..708b33e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptListenerImpTezDag.java
@@ -5,9 +5,9 @@
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
@@ -67,7 +67,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
private static final ContainerTask TASK_FOR_INVALID_JVM = new ContainerTask(
null, true);
-
+
private static ProceedToCompletionResponse COMPLETION_RESPONSE_NO_WAIT =
new ProceedToCompletionResponse(true, true);
@@ -86,10 +86,10 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
// TODO Use this to figure out whether an incoming ping is valid.
private ConcurrentMap<TezTaskAttemptID, ContainerId> attemptToContainerIdMap =
new ConcurrentHashMap<TezTaskAttemptID, ContainerId>();
-
+
private Set<ContainerId> registeredContainers = Collections
.newSetFromMap(new ConcurrentHashMap<ContainerId, Boolean>());
-
+
public TaskAttemptListenerImpTezDag(AppContext context,
TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh,
JobTokenSecretManager jobTokenSecretManager) {
@@ -118,10 +118,10 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
conf.getInt(TezConfiguration.DAG_AM_TASK_LISTENER_THREAD_COUNT,
TezConfiguration.DAG_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT))
.setSecretManager(jobTokenSecretManager).build();
-
+
// Enable service authorization?
if (conf.getBoolean(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
refreshServiceAcls(conf, new MRAMPolicyProvider());
}
@@ -133,7 +133,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
}
- void refreshServiceAcls(Configuration configuration,
+ void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
this.server.refreshServiceAcl(configuration, policyProvider);
}
@@ -153,7 +153,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
public InetSocketAddress getAddress() {
return address;
}
-
+
@Override
public long getProtocolVersion(String protocol, long clientVersion)
throws IOException {
@@ -171,13 +171,13 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
public TezTaskDependencyCompletionEventsUpdate getDependentTasksCompletionEvents(
int fromEventIdx, int maxEvents,
TezTaskAttemptID taskAttemptID) {
-
+
LOG.info("Dependency Completion Events request from " + taskAttemptID
+ ". fromEventID " + fromEventIdx + " maxEvents " + maxEvents);
// TODO: shouldReset is never used. See TT. Ask for Removal.
boolean shouldReset = false;
- TezDependentTaskCompletionEvent[] events =
+ TezDependentTaskCompletionEvent[] events =
context.getDAG().
getVertex(taskAttemptID.getTaskID().getVertexID()).
getTaskAttemptCompletionEvents(fromEventIdx, maxEvents);
@@ -234,22 +234,28 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
}
}
- LOG.info("DEBUG: getTask returning task: " + task);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getTask returning task: " + task);
+ }
return task;
}
@Override
public boolean statusUpdate(TezTaskAttemptID taskAttemptId,
TezTaskStatus taskStatus) throws IOException, InterruptedException {
- LOG.info("DEBUG: " + "Status update from: " + taskAttemptId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Status update from: " + taskAttemptId);
+ }
taskHeartbeatHandler.progressing(taskAttemptId);
pingContainerHeartbeatHandler(taskAttemptId);
TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatus();
taskAttemptStatus.id = taskAttemptId;
// Task sends the updated progress to the TT.
taskAttemptStatus.progress = taskStatus.getProgress();
- LOG.info("DEBUG: " + "Progress of TaskAttempt " + taskAttemptId + " is : "
- + taskStatus.getProgress());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Progress of TaskAttempt " + taskAttemptId + " is : "
+ + taskStatus.getProgress());
+ }
// Task sends the updated state-string to the TT.
taskAttemptStatus.stateString = taskStatus.getStateString();
@@ -267,7 +273,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
// that is the primary storage format inside the AM to avoid multiple
// conversions and unnecessary heap usage.
taskAttemptStatus.counters = taskStatus.getCounters();
-
+
// Map Finish time set by the task (map only)
// TODO CLEANMRXAM - maybe differentiate between map / reduce / types
@@ -366,7 +372,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
/**
* TaskAttempt is reporting that it is in commit_pending and it is waiting for
* the commit Response
- *
+ *
* <br/>
* Commit it a two-phased protocol. First the attempt informs the
* ApplicationMaster that it is
@@ -386,14 +392,14 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
//Ignorable TaskStatus? - since a task will send a LastStatusUpdate
context.getEventHandler().handle(
new TaskAttemptEvent(
- taskAttemptId,
+ taskAttemptId,
TaskAttemptEventType.TA_COMMIT_PENDING)
);
}
/**
* Child checking whether it can commit.
- *
+ *
* <br/>
* Commit is a two-phased protocol. First the attempt informs the
* ApplicationMaster that it is
@@ -411,7 +417,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
pingContainerHeartbeatHandler(taskAttemptId);
DAG job = context.getDAG();
- Task task =
+ Task task =
job.getVertex(taskAttemptId.getTaskID().getVertexID()).
getTask(taskAttemptId.getTaskID());
return task.canCommit(taskAttemptId);
@@ -459,16 +465,16 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
@Override
public ProceedToCompletionResponse
proceedToCompletion(TezTaskAttemptID taskAttemptId) throws IOException {
-
+
// The async nature of the processing combined with the 1 second interval
// between polls (MRTask) implies tasks end up wasting upto 1 second doing
// nothing. Similarly for CA_COMMIT.
-
+
DAG job = context.getDAG();
- Task task =
+ Task task =
job.getVertex(taskAttemptId.getTaskID().getVertexID()).
getTask(taskAttemptId.getTaskID());
-
+
// TODO In-Memory Shuffle
/*
if (task.needsWaitAfterOutputConsumable()) {
@@ -503,7 +509,7 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
*/
return COMPLETION_RESPONSE_NO_WAIT;
}
-
+
@Override
public void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
@@ -539,11 +545,11 @@ public class TaskAttemptListenerImpTezDag extends AbstractService implements
}
registeredContainers.remove(containerId);
}
-
+
private void pingContainerHeartbeatHandler(ContainerId containerId) {
containerHeartbeatHandler.pinged(containerId);
}
-
+
private void pingContainerHeartbeatHandler(TezTaskAttemptID taskAttemptId) {
ContainerId containerId = attemptToContainerIdMap.get(taskAttemptId);
if (containerId != null) {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index d2395ad..de43db0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -558,7 +558,9 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
protected void startRootVertices() {
for (Vertex v : vertices.values()) {
if (v.getInputVerticesCount() == 0) {
- LOG.info("DEBUG: Starting root vertex " + v.getName());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Starting root vertex " + v.getName());
+ }
eventHandler.handle(new VertexEvent(v.getVertexId(),
VertexEventType.V_START));
}
@@ -577,12 +579,10 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
* The only entry point to change the DAG.
*/
public void handle(DAGEvent event) {
- LOG.info("DEBUG: Processing DAGEvent " + event.getDAGId() + " of type "
- + event.getType() + " while in state " + getInternalState()
- + ". Event: " + event);
if (LOG.isDebugEnabled()) {
LOG.debug("Processing DAGEvent " + event.getDAGId() + " of type "
- + event.getType() + " while in state " + getInternalState());
+ + event.getType() + " while in state " + getInternalState()
+ + ". Event: " + event);
}
try {
writeLock.lock();
@@ -658,13 +658,14 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
static DAGState checkJobForCompletion(DAGImpl dag) {
-
- LOG.info("ZZZZ: Checking dag completion"
- + ", numCompletedVertices=" + dag.numCompletedVertices
- + ", numSuccessfulVertices=" + dag.numSuccessfulVertices
- + ", numFailedVertices=" + dag.numFailedVertices
- + ", numKilledVertices=" + dag.numKilledVertices
- + ", numVertices=" + dag.numVertices);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Checking dag completion"
+ + ", numCompletedVertices=" + dag.numCompletedVertices
+ + ", numSuccessfulVertices=" + dag.numSuccessfulVertices
+ + ", numFailedVertices=" + dag.numFailedVertices
+ + ", numKilledVertices=" + dag.numKilledVertices
+ + ", numVertices=" + dag.numVertices);
+ }
if (dag.numFailedVertices > 0) {
dag.setFinishTime();
@@ -870,7 +871,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
for (Vertex v : dag.vertices.values()) {
parseVertexEdges(dag, edgePlans, v);
}
-
+
assignDAGScheduler(dag);
// TODO Metrics
@@ -887,7 +888,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
return dag.finished(DAGState.FAILED);
}
}
-
+
private void assignDAGScheduler(DAGImpl dag) {
boolean isMRR = true;
for(Vertex vertex : dag.vertices.values()) {
@@ -895,15 +896,15 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
if(outVertices == null || outVertices.isEmpty()) {
continue;
}
- if(outVertices.size() > 1 ||
- outVertices.values().iterator().next().getConnectionPattern() !=
+ if(outVertices.size() > 1 ||
+ outVertices.values().iterator().next().getConnectionPattern() !=
EdgeProperty.ConnectionPattern.BIPARTITE) {
// more than 1 output OR single output is not bipartite
isMRR = false;
break;
- }
+ }
}
-
+
if(isMRR) {
LOG.info("Using MRR dag scheduler");
dag.dagScheduler = new DAGSchedulerMRR(dag, dag.eventHandler);
@@ -1110,10 +1111,11 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
public DAGState transition(DAGImpl job, DAGEvent event) {
DAGEventVertexCompleted vertexEvent = (DAGEventVertexCompleted) event;
- LOG.info("DEBUG: Received a vertex completion event"
- + ", vertexId=" + vertexEvent.getVertexId()
- + ", vertexState=" + vertexEvent.getVertexState());
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received a vertex completion event"
+ + ", vertexId=" + vertexEvent.getVertexId()
+ + ", vertexState=" + vertexEvent.getVertexState());
+ }
Vertex vertex = job.vertices.get(vertexEvent.getVertexId());
job.numCompletedVertices++;
if (vertexEvent.getVertexState() == VertexState.SUCCEEDED) {
@@ -1124,12 +1126,14 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
vertexKilled(job, vertex);
}
- LOG.info("ZZZZ: Vertex completed."
- + ", numCompletedVertices=" + job.numCompletedVertices
- + ", numSuccessfulVertices=" + job.numSuccessfulVertices
- + ", numFailedVertices=" + job.numFailedVertices
- + ", numKilledVertices=" + job.numKilledVertices
- + ", numVertices=" + job.numVertices);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Vertex completed."
+ + ", numCompletedVertices=" + job.numCompletedVertices
+ + ", numSuccessfulVertices=" + job.numSuccessfulVertices
+ + ", numFailedVertices=" + job.numFailedVertices
+ + ", numKilledVertices=" + job.numKilledVertices
+ + ", numVertices=" + job.numVertices);
+ }
job.dagScheduler.vertexCompleted(vertex);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index c14ba8a..94e5161 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -109,7 +109,7 @@ public class TaskAttemptImpl implements TaskAttempt,
private static final Log LOG = LogFactory.getLog(TaskAttemptImpl.class);
private static final String LINE_SEPARATOR = System
.getProperty("line.separator");
-
+
static final TezCounters EMPTY_COUNTERS = new TezCounters();
protected final TezConfiguration conf;
@@ -147,22 +147,22 @@ public class TaskAttemptImpl implements TaskAttempt,
protected final boolean isRescheduled;
protected String processorName;
-
+
protected static final FailedTransitionHelper FAILED_HELPER =
new FailedTransitionHelper();
-
+
protected static final KilledTransitionHelper KILLED_HELPER =
new KilledTransitionHelper();
private static SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent>
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION =
new DiagnosticInformationUpdater();
-
+
private static SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent>
STATUS_UPDATER = new StatusUpdaterTransition();
private final StateMachine<TaskAttemptStateInternal, TaskAttemptEventType, TaskAttemptEvent> stateMachine;
-
+
private static StateMachineFactory
<TaskAttemptImpl, TaskAttemptStateInternal, TaskAttemptEventType, TaskAttemptEvent>
stateMachineFactory
@@ -174,7 +174,7 @@ public class TaskAttemptImpl implements TaskAttempt,
.addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.NEW, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
.addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_FAIL_REQUEST, new TerminateTransition(FAILED_HELPER))
.addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_KILL_REQUEST, new TerminateTransition(KILLED_HELPER))
-
+
.addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_STARTED_REMOTELY, new StartedTransition())
.addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.START_WAIT, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
.addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptEventType.TA_FAIL_REQUEST, new TerminatedBeforeRunningTransition(FAILED_HELPER))
@@ -182,7 +182,7 @@ public class TaskAttemptImpl implements TaskAttempt,
.addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptEventType.TA_NODE_FAILED, new NodeFailedBeforeRunningTransition())
.addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptEventType.TA_CONTAINER_TERMINATING, new ContainerTerminatingBeforeRunningTransition())
.addTransition(TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATED, new ContainerCompletedBeforeRunningTransition())
-
+
.addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_STATUS_UPDATE, STATUS_UPDATER)
.addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
.addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE, new OutputConsumableTransition()) //Optional, may not come in for all tasks.
@@ -195,7 +195,7 @@ public class TaskAttemptImpl implements TaskAttempt,
.addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptEventType.TA_NODE_FAILED, new TerminatedWhileRunningTransition(KILLED_HELPER))
.addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptEventType.TA_CONTAINER_TERMINATING, new TerminatedWhileRunningTransition(FAILED_HELPER))
.addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATED, new ContainerCompletedWhileRunningTransition())
-
+
.addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_STATUS_UPDATE, STATUS_UPDATER)
.addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
.addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptStateInternal.OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE) // Stuck RPC. The client retries in a loop.
@@ -227,17 +227,17 @@ public class TaskAttemptImpl implements TaskAttempt,
.addTransition(TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_CONTAINER_TERMINATED, new ContainerCompletedWhileTerminating())
.addTransition(TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
.addTransition(TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptStateInternal.KILL_IN_PROGRESS, EnumSet.of(TaskAttemptEventType.TA_STARTED_REMOTELY, TaskAttemptEventType.TA_STATUS_UPDATE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED, TaskAttemptEventType.TA_TIMED_OUT, TaskAttemptEventType.TA_FAIL_REQUEST, TaskAttemptEventType.TA_KILL_REQUEST, TaskAttemptEventType.TA_NODE_FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATING, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES))
-
+
.addTransition(TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATED, new ContainerCompletedWhileTerminating())
.addTransition(TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
.addTransition(TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptStateInternal.FAIL_IN_PROGRESS, EnumSet.of(TaskAttemptEventType.TA_STARTED_REMOTELY, TaskAttemptEventType.TA_STATUS_UPDATE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED, TaskAttemptEventType.TA_TIMED_OUT, TaskAttemptEventType.TA_FAIL_REQUEST, TaskAttemptEventType.TA_KILL_REQUEST, TaskAttemptEventType.TA_NODE_FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATING, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES))
-
+
.addTransition(TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
.addTransition(TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.KILLED, EnumSet.of(TaskAttemptEventType.TA_STARTED_REMOTELY, TaskAttemptEventType.TA_STATUS_UPDATE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED, TaskAttemptEventType.TA_TIMED_OUT, TaskAttemptEventType.TA_FAIL_REQUEST, TaskAttemptEventType.TA_KILL_REQUEST, TaskAttemptEventType.TA_NODE_FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATING, TaskAttemptEventType.TA_CONTAINER_TERMINATED, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES))
.addTransition(TaskAttemptStateInternal.FAILED, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
.addTransition(TaskAttemptStateInternal.FAILED, TaskAttemptStateInternal.FAILED, EnumSet.of(TaskAttemptEventType.TA_STARTED_REMOTELY, TaskAttemptEventType.TA_STATUS_UPDATE, TaskAttemptEventType.TA_OUTPUT_CONSUMABLE, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED, TaskAttemptEventType.TA_TIMED_OUT, TaskAttemptEventType.TA_FAIL_REQUEST, TaskAttemptEventType.TA_KILL_REQUEST, TaskAttemptEventType.TA_NODE_FAILED, TaskAttemptEventType.TA_CONTAINER_TERMINATING, TaskAttemptEventType.TA_CONTAINER_TERMINATED, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES))
-
+
// How will duplicate history events be handled ?
// TODO Maybe consider not failing REDUCE tasks in this case. Also, MAP_TASKS in case there's only one phase in the job.
.addTransition(TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.SUCCEEDED, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
@@ -245,21 +245,21 @@ public class TaskAttemptImpl implements TaskAttempt,
.addTransition(TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_NODE_FAILED, new TerminatedAfterSuccessTransition(KILLED_HELPER))
.addTransition(TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURES, new TerminatedAfterSuccessTransition(FAILED_HELPER))
.addTransition(TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.SUCCEEDED, EnumSet.of(TaskAttemptEventType.TA_TIMED_OUT, TaskAttemptEventType.TA_FAIL_REQUEST, TaskAttemptEventType.TA_CONTAINER_TERMINATING, TaskAttemptEventType.TA_CONTAINER_TERMINATED))
-
-
+
+
.installTopology();
-
+
// TODO Remove TaskAttemptListener from the constructor.
@SuppressWarnings("rawtypes")
public TaskAttemptImpl(TezTaskID taskId, int attemptNumber, EventHandler eventHandler,
- TaskAttemptListener tal, int partition,
+ TaskAttemptListener tal, int partition,
TezConfiguration conf,
Token<JobTokenIdentifier> jobToken, Credentials credentials, Clock clock,
TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
String processorName, TaskLocationHint locationHint,
Resource resource, Map<String, LocalResource> localResources,
- Map<String, String> environment,
+ Map<String, String> environment,
String javaOpts, boolean isRescheduled) {
ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
this.readLock = rwLock.readLock();
@@ -286,34 +286,34 @@ public class TaskAttemptImpl implements TaskAttempt,
this.javaOpts = javaOpts;
this.isRescheduled = isRescheduled;
}
-
+
@Override
public TezTaskAttemptID getID() {
return attemptId;
}
-
+
@Override
public TezVertexID getVertexID() {
return attemptId.getTaskID().getVertexID();
}
-
+
@Override
public TezDAGID getDAGID() {
return getVertexID().getDAGId();
}
-
+
TezTaskContext createRemoteTask() {
Vertex vertex = getTask().getVertex();
DAG dag = vertex.getDAG();
// TODO TEZ-50 user and jobname
- return new TezEngineTaskContext(getID(), dag.getUserName(),
+ return new TezEngineTaskContext(getID(), dag.getUserName(),
dag.getName(), getTask()
.getVertex().getName(), processorName,
vertex.getInputSpecList(), vertex.getOutputSpecList());
}
-
+
@Override
public TaskAttemptReport getReport() {
TaskAttemptReport result = Records.newRecord(TaskAttemptReport.class);
@@ -322,7 +322,7 @@ public class TaskAttemptImpl implements TaskAttempt,
result.setTaskAttemptId(attemptId);
//take the LOCAL state of attempt
//DO NOT take from reportedStatus
-
+
result.setTaskAttemptState(getState());
result.setProgress(reportedStatus.progress);
result.setStartTime(launchTime);
@@ -522,11 +522,9 @@ public class TaskAttemptImpl implements TaskAttempt,
public void handle(TaskAttemptEvent event) {
if (LOG.isDebugEnabled()) {
LOG.debug("Processing TaskAttemptEvent " + event.getTaskAttemptID()
- + " of type " + event.getType());
+ + " of type " + event.getType() + " while in state "
+ + getInternalState() + ". Event: " + event);
}
- LOG.info("DEBUG: Processing TaskAttemptEvent " + event.getTaskAttemptID()
- + " of type " + event.getType() + " while in state "
- + getInternalState() + ". Event: " + event);
writeLock.lock();
try {
final TaskAttemptStateInternal oldState = getInternalState();
@@ -536,8 +534,8 @@ public class TaskAttemptImpl implements TaskAttempt,
LOG.error("Can't handle this event at current state for "
+ this.attemptId, e);
eventHandler.handle(new DAGEventDiagnosticsUpdate(
- this.attemptId.getTaskID().getVertexID().getDAGId(),
- "Invalid event " + event.getType() +
+ this.attemptId.getTaskID().getVertexID().getDAGId(),
+ "Invalid event " + event.getType() +
" on TaskAttempt " + this.attemptId));
eventHandler.handle(
new DAGEvent(
@@ -546,7 +544,7 @@ public class TaskAttemptImpl implements TaskAttempt,
);
}
if (oldState != getInternalState()) {
- LOG.info(attemptId + " TaskAttempt Transitioned from "
+ LOG.info(attemptId + " TaskAttempt Transitioned from "
+ oldState + " to "
+ getInternalState());
}
@@ -554,7 +552,7 @@ public class TaskAttemptImpl implements TaskAttempt,
writeLock.unlock();
}
}
-
+
@VisibleForTesting
public TaskAttemptStateInternal getInternalState() {
readLock.lock();
@@ -562,7 +560,7 @@ public class TaskAttemptImpl implements TaskAttempt,
return stateMachine.getCurrentState();
} finally {
readLock.unlock();
- }
+ }
}
private static TaskAttemptState getExternalState(
@@ -590,7 +588,7 @@ public class TaskAttemptImpl implements TaskAttempt,
+ smState);
}
}
-
+
@Override
public boolean getIsRescheduled() {
return isRescheduled;
@@ -612,7 +610,7 @@ public class TaskAttemptImpl implements TaskAttempt,
// TOOD Merge some of these JobCounter events.
private static DAGEventCounterUpdate createJobCounterUpdateEventTALaunched(
TaskAttemptImpl ta) {
- DAGEventCounterUpdate jce =
+ DAGEventCounterUpdate jce =
new DAGEventCounterUpdate(
ta.getDAGID()
);
@@ -622,8 +620,8 @@ public class TaskAttemptImpl implements TaskAttempt,
private static DAGEventCounterUpdate createJobCounterUpdateEventSlotMillis(
TaskAttemptImpl ta) {
- DAGEventCounterUpdate jce =
- new DAGEventCounterUpdate(
+ DAGEventCounterUpdate jce =
+ new DAGEventCounterUpdate(
ta.getDAGID()
);
@@ -635,7 +633,7 @@ public class TaskAttemptImpl implements TaskAttempt,
private static DAGEventCounterUpdate createJobCounterUpdateEventTATerminated(
TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted,
TaskAttemptStateInternal taState) {
- DAGEventCounterUpdate jce =
+ DAGEventCounterUpdate jce =
new DAGEventCounterUpdate(
taskAttempt.getDAGID());
@@ -687,16 +685,16 @@ public class TaskAttemptImpl implements TaskAttempt,
taskAttempt.finishTime,
taskAttempt.containerNodeId == null ? "UNKNOWN"
: taskAttempt.containerNodeId.getHost(),
- taskAttempt.containerNodeId == null ? -1
- : taskAttempt.containerNodeId.getPort(),
- taskAttempt.nodeRackName == null ? "UNKNOWN"
+ taskAttempt.containerNodeId == null ? -1
+ : taskAttempt.containerNodeId.getPort(),
+ taskAttempt.nodeRackName == null ? "UNKNOWN"
: taskAttempt.nodeRackName,
StringUtils.join(
LINE_SEPARATOR, taskAttempt.getDiagnostics()), taskAttempt
.getProgressSplitBlock().burst());
return tauce;
}
-
+
// TODO Incorporate MAPREDUCE-4838
private JobHistoryEvent createTaskAttemptStartedEvent() {
TaskAttemptStartedEvent tase = new TaskAttemptStartedEvent(
@@ -707,7 +705,7 @@ public class TaskAttemptImpl implements TaskAttempt,
}
*/
-
+
// private WrappedProgressSplitsBlock getProgressSplitBlock() {
// return null;
// // TODO
@@ -725,7 +723,7 @@ public class TaskAttemptImpl implements TaskAttempt,
// }
// */
// }
-
+
private void updateProgressSplits() {
// double newProgress = reportedStatus.progress;
// newProgress = Math.max(Math.min(newProgress, 1.0D), 0.0D);
@@ -737,7 +735,7 @@ public class TaskAttemptImpl implements TaskAttempt,
// if (splitsBlock != null) {
// long now = clock.getTime();
// long start = getLaunchTime();
-//
+//
// if (start == 0)
// return;
//
@@ -767,7 +765,7 @@ public class TaskAttemptImpl implements TaskAttempt,
// }
// }
}
-
+
// private void maybeSendSpeculatorContainerRequired() {
// if (!speculatorContainerRequestSent) {
// sendEvent(new SpeculatorEvent(getID().getTaskID(), +1));
@@ -781,10 +779,10 @@ public class TaskAttemptImpl implements TaskAttempt,
// speculatorContainerRequestSent = false;
// }
// }
-
+
private void sendTaskAttemptCleanupEvent() {
-// TaskAttemptContext taContext =
-// new TaskAttemptContextImpl(this.conf,
+// TaskAttemptContext taContext =
+// new TaskAttemptContextImpl(this.conf,
// TezMRTypeConverter.fromTez(this.attemptId));
// sendEvent(new TaskCleanupEvent(this.attemptId, this.committer, taContext));
}
@@ -825,7 +823,7 @@ public class TaskAttemptImpl implements TaskAttempt,
protected void logJobHistoryAttemptFinishedEvent(TaskAttemptStateInternal state) {
//Log finished events only if an attempt started.
if (getLaunchTime() == 0) return;
-
+
TaskAttemptFinishedEvent finishEvt = new TaskAttemptFinishedEvent(
attemptId, getTask().getVertex().getName(),
getFinishTime(), TaskAttemptState.SUCCEEDED, "",
@@ -901,7 +899,7 @@ public class TaskAttemptImpl implements TaskAttempt,
ta.localResources, remoteTaskContext, ta,
ta.credentials, ta.jobToken, hostArray,
rackArray,
- scheduleEvent.getPriority(), ta.environment, //ta.javaOpts,
+ scheduleEvent.getPriority(), ta.environment, //ta.javaOpts,
ta.conf);
ta.sendEvent(launchRequestEvent);
}
@@ -1269,7 +1267,7 @@ public class TaskAttemptImpl implements TaskAttempt,
return TaskEventType.T_ATTEMPT_KILLED;
}
}
-
+
@Override
public String toString() {
return getID().toString();
@@ -1284,10 +1282,10 @@ public class TaskAttemptImpl implements TaskAttempt,
public Map<String, String> getEnvironment() {
return this.environment;
}
-
+
@Override
public String getJavaOpts() {
- return this.javaOpts;
+ return this.javaOpts;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 575f32c..f295caf 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -107,68 +107,68 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
protected Resource taskResource;
protected Map<String, LocalResource> localResources;
protected Map<String, String> environment;
-
+
// counts the number of attempts that are either running or in a state where
// they will come to be running when they get a Container
private int numberUncompletedAttempts = 0;
private boolean historyTaskStartGenerated = false;
-
- private static final SingleArcTransition<TaskImpl, TaskEvent>
+
+ private static final SingleArcTransition<TaskImpl, TaskEvent>
ATTEMPT_KILLED_TRANSITION = new AttemptKilledTransition();
- private static final SingleArcTransition<TaskImpl, TaskEvent>
+ private static final SingleArcTransition<TaskImpl, TaskEvent>
KILL_TRANSITION = new KillTransition();
private static final StateMachineFactory
- <TaskImpl, TaskStateInternal, TaskEventType, TaskEvent>
- stateMachineFactory
+ <TaskImpl, TaskStateInternal, TaskEventType, TaskEvent>
+ stateMachineFactory
= new StateMachineFactory<TaskImpl, TaskStateInternal, TaskEventType, TaskEvent>
(TaskStateInternal.NEW)
// define the state machine of Task
// Transitions from NEW state
- .addTransition(TaskStateInternal.NEW, TaskStateInternal.SCHEDULED,
+ .addTransition(TaskStateInternal.NEW, TaskStateInternal.SCHEDULED,
TaskEventType.T_SCHEDULE, new InitialScheduleTransition())
- .addTransition(TaskStateInternal.NEW, TaskStateInternal.KILLED,
+ .addTransition(TaskStateInternal.NEW, TaskStateInternal.KILLED,
TaskEventType.T_KILL, new KillNewTransition())
// Transitions from SCHEDULED state
//when the first attempt is launched, the task state is set to RUNNING
- .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.RUNNING,
+ .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.RUNNING,
TaskEventType.T_ATTEMPT_LAUNCHED, new LaunchTransition())
- .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.KILL_WAIT,
+ .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.KILL_WAIT,
TaskEventType.T_KILL, KILL_TRANSITION)
- .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.SCHEDULED,
+ .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.SCHEDULED,
TaskEventType.T_ATTEMPT_KILLED, ATTEMPT_KILLED_TRANSITION)
- .addTransition(TaskStateInternal.SCHEDULED,
- EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.FAILED),
- TaskEventType.T_ATTEMPT_FAILED,
+ .addTransition(TaskStateInternal.SCHEDULED,
+ EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.FAILED),
+ TaskEventType.T_ATTEMPT_FAILED,
new AttemptFailedTransition())
-
+
// Transitions from RUNNING state
- .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
+ .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
TaskEventType.T_ATTEMPT_LAUNCHED) //more attempts may start later
- // This is an optional event.
+ // This is an optional event.
.addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
TaskEventType.T_ATTEMPT_OUTPUT_CONSUMABLE,
new AttemptProcessingCompleteTransition())
- .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
+ .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
TaskEventType.T_ATTEMPT_COMMIT_PENDING,
new AttemptCommitPendingTransition())
.addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
TaskEventType.T_ADD_SPEC_ATTEMPT, new RedundantScheduleTransition())
- .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.SUCCEEDED,
+ .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.SUCCEEDED,
TaskEventType.T_ATTEMPT_SUCCEEDED,
new AttemptSucceededTransition())
- .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
+ .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
TaskEventType.T_ATTEMPT_KILLED,
ATTEMPT_KILLED_TRANSITION)
- .addTransition(TaskStateInternal.RUNNING,
- EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.FAILED),
+ .addTransition(TaskStateInternal.RUNNING,
+ EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.FAILED),
TaskEventType.T_ATTEMPT_FAILED,
new AttemptFailedTransition())
- .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.KILL_WAIT,
+ .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.KILL_WAIT,
TaskEventType.T_KILL, KILL_TRANSITION)
// Transitions from KILL_WAIT state
@@ -203,7 +203,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
EnumSet.of(TaskEventType.T_ADD_SPEC_ATTEMPT,
TaskEventType.T_ATTEMPT_LAUNCHED))
- // Transitions from FAILED state
+ // Transitions from FAILED state
.addTransition(TaskStateInternal.FAILED, TaskStateInternal.FAILED,
EnumSet.of(TaskEventType.T_KILL,
TaskEventType.T_ADD_SPEC_ATTEMPT))
@@ -221,7 +221,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
// TODO: Recovery
/*
- // By default, the next TaskAttempt number is zero. Changes during recovery
+ // By default, the next TaskAttempt number is zero. Changes during recovery
protected int nextAttemptNumber = 0;
private List<TaskAttemptInfo> taskAttemptsFromPreviousGeneration =
new ArrayList<TaskAttemptInfo>();
@@ -239,10 +239,10 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
new RecoverdAttemptsComparator();
*/
-
+
private TezTaskAttemptID outputConsumableAttempt;
private boolean outputConsumableAttemptSuccessSent = false;
-
+
//should be set to one which comes first
//saying COMMIT_PENDING
private TezTaskAttemptID commitAttempt;
@@ -272,10 +272,10 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
Token<JobTokenIdentifier> jobToken,
Credentials credentials, Clock clock,
// TODO Recovery
- //Map<TezTaskID, TaskInfo> completedTasksFromPreviousRun,
+ //Map<TezTaskID, TaskInfo> completedTasksFromPreviousRun,
//int startCount,
// TODO Metrics
- //MRAppMetrics metrics,
+ //MRAppMetrics metrics,
TaskHeartbeatHandler thh, AppContext appContext,
String processorName,
boolean leafVertex, TaskLocationHint locationHint, Resource resource,
@@ -289,7 +289,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
writeLock = readWriteLock.writeLock();
this.attempts = Collections.emptyMap();
// TODO TEZ-47 get from conf or API
- maxAttempts = this.conf.getInt(TezConfiguration.DAG_MAX_TASK_ATTEMPTS,
+ maxAttempts = this.conf.getInt(TezConfiguration.DAG_MAX_TASK_ATTEMPTS,
TezConfiguration.DAG_MAX_TASK_ATTEMPTS_DEFAULT);
taskId = new TezTaskID(vertexId, partition);
this.partition = partition;
@@ -360,7 +360,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
if (attempts.size() <= 1) {
return attempts;
}
-
+
Map<TezTaskAttemptID, TaskAttempt> result
= new LinkedHashMap<TezTaskAttemptID, TaskAttempt>();
result.putAll(attempts);
@@ -385,7 +385,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
public Vertex getVertex() {
return appContext.getDAG().getVertex(taskId.getVertexID());
}
-
+
@Override
public TezTaskID getTaskId() {
return taskId;
@@ -424,12 +424,12 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
}
report.setSuccessfulAttempt(successfulAttempt);
-
+
for (TaskAttempt att : attempts.values()) {
String prefix = "AttemptID:" + att.getID() + " Info:";
for (CharSequence cs : att.getDiagnostics()) {
report.addDiagnostics(prefix + cs);
-
+
}
}
@@ -482,7 +482,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
return stateMachine.getCurrentState();
} finally {
readLock.unlock();
- }
+ }
}
private static TaskState getExternalState(TaskStateInternal smState) {
@@ -490,7 +490,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
return TaskState.KILLED;
} else {
return TaskState.valueOf(smState.name());
- }
+ }
}
//this is always called in read/write lock
@@ -530,7 +530,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
}
return finishTime;
}
-
+
private TaskStateInternal finished(TaskStateInternal finalState) {
if (getInternalState() == TaskStateInternal.RUNNING) {
// TODO Metrics
@@ -552,7 +552,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
case KILLED:
continue;
default:
- }
+ }
if (result == null) {
result = at; //The first time around
}
@@ -589,8 +589,8 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
return false;
}
}
-
-
+
+
@Override
public TezTaskAttemptID getOutputConsumableAttempt() {
readLock.lock();
@@ -608,7 +608,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
appContext, processorName, locationHint, taskResource,
localResources, environment, javaOpts, (failedAttempts>0));
}
-
+
protected TaskAttempt getSuccessfulAttempt() {
readLock.lock();
try {
@@ -631,7 +631,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
case 0:
attempts = Collections.singletonMap(attempt.getID(), attempt);
break;
-
+
case 1:
Map<TezTaskAttemptID, TaskAttempt> newAttempts
= new LinkedHashMap<TezTaskAttemptID, TaskAttempt>(maxAttempts);
@@ -663,17 +663,15 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
// picture in mind
eventHandler.handle(new DAGEventSchedulerUpdate(
DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, attempt));
-
+
}
@Override
public void handle(TaskEvent event) {
- LOG.info("DEBUG: Processing TaskEvent " + event.getTaskID() + " of type "
- + event.getType() + " while in state " + getInternalState()
- + ". Event: " + event);
if (LOG.isDebugEnabled()) {
LOG.debug("Processing TaskEvent " + event.getTaskID() + " of type "
- + event.getType());
+ + event.getType() + " while in state " + getInternalState()
+ + ". Event: " + event);
}
try {
writeLock.lock();
@@ -698,7 +696,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
protected void internalError(TaskEventType type) {
LOG.error("Invalid event " + type + " on Task " + this.taskId);
eventHandler.handle(new DAGEventDiagnosticsUpdate(
- this.taskId.getVertexID().getDAGId(), "Invalid event " + type +
+ this.taskId.getVertexID().getDAGId(), "Invalid event " + type +
" on Task " + this.taskId));
eventHandler.handle(new DAGEvent(this.taskId.getVertexID().getDAGId(),
DAGEventType.INTERNAL_ERROR));
@@ -719,19 +717,19 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
}
}
}
- if (attempt.getNodeHttpAddress() != null) {
-
+ if (attempt.getNodeHttpAddress() != null) {
+
String scheme = (encryptedShuffle) ? "https://" : "http://";
String url = scheme
+ attempt.getNodeHttpAddress().split(":")[0] + ":"
+ attempt.getShufflePort();
-
-
-
+
+
+
int runTime = 0;
if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() != 0)
runTime = (int) (attempt.getFinishTime() - attempt.getLaunchTime());
-
+
TezDependentTaskCompletionEvent tce = new TezDependentTaskCompletionEvent(
-1, attemptId, status, url, runTime);
@@ -740,7 +738,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
eventHandler.handle(new VertexEventTaskAttemptCompleted(tce));
}
}
-
+
// always called inside a transition, in turn inside the Write Lock
private void handleTaskAttemptCompletion(TezTaskAttemptID attemptId,
TezDependentTaskCompletionEvent.Status status) {
@@ -759,7 +757,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
task.getCounters());
return tfe;
}
-
+
private static TaskFailedEvent createTaskFailedEvent(TaskImpl task, List<String> diag, TaskStateInternal taskState, TezTaskAttemptID taId) {
StringBuilder errorSb = new StringBuilder();
if (diag != null) {
@@ -778,7 +776,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
return taskFailedEvent;
}
*/
-
+
private static void unSucceed(TaskImpl task) {
task.commitAttempt = null;
task.successfulAttempt = null;
@@ -801,7 +799,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
this.eventHandler.handle(new DAGHistoryEvent(
taskId.getVertexID().getDAGId(), startEvt));
}
-
+
protected void logJobHistoryTaskFinishedEvent() {
// FIXME need to handle getting finish time as this function
// is called from within a transition
@@ -811,7 +809,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
this.eventHandler.handle(new DAGHistoryEvent(
taskId.getVertexID().getDAGId(), finishEvt));
}
-
+
protected void logJobHistoryTaskFailedEvent(TaskState finalState) {
TaskFinishedEvent finishEvt = new TaskFinishedEvent(taskId,
getVertex().getName(), clock.getTime(), finalState, getCounters());
@@ -859,15 +857,19 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
TezDependentTaskCompletionEvent.Status.SUCCEEDED);
task.outputConsumableAttempt = attemptId;
task.outputConsumableAttemptSuccessSent = true;
- LOG.info("DEBUG: TezTaskAttemptID: " + attemptId
- + " set as the OUTPUT_READY attempt");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("TezTaskAttemptID: " + attemptId
+ + " set as the OUTPUT_READY attempt");
+ }
} else {
// Nothing to do. This task will eventually be told to die, or will be
// killed.
- LOG.info("DEBUG: TezTaskAttemptID: "
- + attemptId
- + " reporting OUTPUT_READY. Will be asked to die since another attempt "
- + task.outputConsumableAttempt + " already has output ready");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("TezTaskAttemptID: "
+ + attemptId + " reporting OUTPUT_READY."
+ + " Will be asked to die since another attempt "
+ + task.outputConsumableAttempt + " already has output ready");
+ }
task.eventHandler.handle(new TaskAttemptEventKillRequest(attemptId,
"Alternate attemptId already serving output"));
}
@@ -899,12 +901,12 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
}
}
- private static class AttemptSucceededTransition
+ private static class AttemptSucceededTransition
implements SingleArcTransition<TaskImpl, TaskEvent> {
@Override
public void transition(TaskImpl task, TaskEvent event) {
task.handleTaskAttemptCompletion(
- ((TaskEventTAUpdate) event).getTaskAttemptID(),
+ ((TaskEventTAUpdate) event).getTaskAttemptID(),
TezDependentTaskCompletionEvent.Status.SUCCEEDED);
task.finishedAttempts++;
--task.numberUncompletedAttempts;
@@ -937,7 +939,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
@Override
public void transition(TaskImpl task, TaskEvent event) {
task.handleTaskAttemptCompletion(
- ((TaskEventTAUpdate) event).getTaskAttemptID(),
+ ((TaskEventTAUpdate) event).getTaskAttemptID(),
TezDependentTaskCompletionEvent.Status.KILLED);
task.finishedAttempts++;
--task.numberUncompletedAttempts;
@@ -956,7 +958,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
@Override
public TaskStateInternal transition(TaskImpl task, TaskEvent event) {
task.handleTaskAttemptCompletion(
- ((TaskEventTAUpdate) event).getTaskAttemptID(),
+ ((TaskEventTAUpdate) event).getTaskAttemptID(),
TezDependentTaskCompletionEvent.Status.KILLED);
task.finishedAttempts++;
// check whether all attempts are finished
@@ -997,7 +999,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
task.finishedAttempts++;
if (task.failedAttempts < task.maxAttempts) {
task.handleTaskAttemptCompletion(
- ((TaskEventTAUpdate) event).getTaskAttemptID(),
+ ((TaskEventTAUpdate) event).getTaskAttemptID(),
TezDependentTaskCompletionEvent.Status.FAILED);
// we don't need a new event if we already have a spare
if (--task.numberUncompletedAttempts == 0
@@ -1006,9 +1008,9 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
}
} else {
task.handleTaskAttemptCompletion(
- ((TaskEventTAUpdate) event).getTaskAttemptID(),
+ ((TaskEventTAUpdate) event).getTaskAttemptID(),
TezDependentTaskCompletionEvent.Status.TIPFAILED);
-
+
if (task.historyTaskStartGenerated) {
task.logJobHistoryTaskFailedEvent(TaskState.FAILED);
} else {
@@ -1046,7 +1048,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
LOG.error("Unexpected event for task of leaf vertex " + event.getType());
task.internalError(event.getType());
}
-
+
// tell the job about the rescheduling
task.eventHandler.handle(
new VertexEventTaskReschedule(task.taskId));
@@ -1085,16 +1087,16 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
// tell the job about the rescheduling
unSucceed(task);
task.handleTaskAttemptCompletion(
- attemptId,
+ attemptId,
TezDependentTaskCompletionEvent.Status.KILLED);
task.eventHandler.handle(new VertexEventTaskReschedule(task.taskId));
- // typically we are here because this map task was run on a bad node and
+ // typically we are here because this map task was run on a bad node and
// we want to reschedule it on a different node.
- // Depending on whether there are previous failed attempts or not this
+ // Depending on whether there are previous failed attempts or not this
// can SCHEDULE or RESCHEDULE the container allocate request. If this
// SCHEDULE's then the dataLocal hosts of this taskAttempt will be used
- // from the map splitInfo. So the bad node might be sent as a location
- // to the RM. But the RM would ignore that just like it would ignore
+ // from the map splitInfo. So the bad node might be sent as a location
+ // to the RM. But the RM would ignore that just like it would ignore
// currently pending container requests affinitized to bad nodes.
task.addAndScheduleAttempt();
return TaskStateInternal.SCHEDULED;
@@ -1105,11 +1107,11 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
}
}
- private static class KillNewTransition
+ private static class KillNewTransition
implements SingleArcTransition<TaskImpl, TaskEvent> {
@Override
public void transition(TaskImpl task, TaskEvent event) {
-
+
if (task.historyTaskStartGenerated) {
task.logJobHistoryTaskFailedEvent(TaskState.KILLED);
} else {
@@ -1131,7 +1133,7 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
}
}
- private static class KillTransition
+ private static class KillTransition
implements SingleArcTransition<TaskImpl, TaskEvent> {
@Override
public void transition(TaskImpl task, TaskEvent event) {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index b06e264..c8ee314 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -599,12 +599,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
* The only entry point to change the Vertex.
*/
public void handle(VertexEvent event) {
- LOG.info("DEBUG: Processing VertexEvent " + event.getVertexId()
- + " of type " + event.getType() + " while in state "
- + getInternalState() + ". Event: " + event);
if (LOG.isDebugEnabled()) {
- LOG.debug("Processing VertexEvent " + event.getVertexId() + " of type "
- + event.getType() + " while in state " + getInternalState());
+ LOG.debug("Processing VertexEvent " + event.getVertexId()
+ + " of type " + event.getType() + " while in state "
+ + getInternalState() + ". Event: " + event);
}
try {
writeLock.lock();
@@ -631,9 +629,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
finally {
writeLock.unlock();
}
- LOG.info("DEBUG: Finished processing VertexEvent " + event.getVertexId()
- + " of type " + event.getType() + " while in state "
- + getInternalState() + ". Event: " + event);
}
private VertexState getInternalState() {
@@ -699,12 +694,13 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
static VertexState checkVertexForCompletion(VertexImpl vertex) {
//check for vertex failure first
-
- LOG.info("ZZZZ: checking for vertex completion"
- + ", failedTaskCount=" + vertex.failedTaskCount
- + ", killedTaskCount=" + vertex.killedTaskCount
- + ", successfulTaskCount=" + vertex.succeededTaskCount
- + ", completedTaskCount=" + vertex.completedTaskCount);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Checking for vertex completion"
+ + ", failedTaskCount=" + vertex.failedTaskCount
+ + ", killedTaskCount=" + vertex.killedTaskCount
+ + ", successfulTaskCount=" + vertex.succeededTaskCount
+ + ", completedTaskCount=" + vertex.completedTaskCount);
+ }
if (vertex.failedTaskCount > 0) {
vertex.setFinishTime();
@@ -1287,8 +1283,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
InputSpec inputSpec = new InputSpec(entry.getKey().getName(),
entry.getKey().getTotalTasks(),
entry.getValue().getInputClass());
- LOG.info("DEBUG: For vertex : " + this.getName()
- + ", Using InputSpec : " + inputSpec);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("For vertex : " + this.getName()
+ + ", Using InputSpec : " + inputSpec);
+ }
// TODO DAGAM This should be based on the edge type.
inputSpecList.add(inputSpec);
}
@@ -1304,8 +1302,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
OutputSpec outputSpec = new OutputSpec(entry.getKey().getName(),
entry.getKey().getTotalTasks(),
entry.getValue().getOutputClass());
- LOG.info("DEBUG: For vertex : " + this.getName()
- + ", Using OutputSpec : " + outputSpec);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("For vertex : " + this.getName()
+ + ", Using OutputSpec : " + outputSpec);
+ }
// TODO DAGAM This should be based on the edge type.
outputSpecList.add(outputSpec);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9feab053/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index 58e9d50..bab02da 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -59,7 +59,7 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
public class AMContainerImpl implements AMContainer {
private static final Log LOG = LogFactory.getLog(AMContainerImpl.class);
-
+
private final ReadLock readLock;
private final WriteLock writeLock;
private final ContainerId containerId;
@@ -75,10 +75,10 @@ public class AMContainerImpl implements AMContainer {
// TODO Maybe this should be pulled from the TaskAttempt.s
private final Map<TezTaskAttemptID, TezTaskContext> remoteTaskMap =
new HashMap<TezTaskAttemptID, TezTaskContext>();
-
+
// TODO ?? Convert to list and hash.
-
- private int shufflePort;
+
+ private int shufflePort;
private long idleTimeBetweenTasks = 0;
private long lastTaskFinishTime;
@@ -92,16 +92,16 @@ public class AMContainerImpl implements AMContainer {
private TezTaskAttemptID runningAttempt;
private List<TezTaskAttemptID> failedAssignments;
private TezTaskAttemptID pullAttempt;
-
+
private AMContainerTask noAllocationContainerTask;
-
+
private static final AMContainerTask NO_MORE_TASKS = new AMContainerTask(
true, null);
private static final AMContainerTask WAIT_TASK = new AMContainerTask(false,
null);
-
+
private boolean inError = false;
-
+
private ContainerLaunchContext clc;
// TODO Consider registering with the TAL, instead of the TAL pulling.
@@ -115,8 +115,8 @@ public class AMContainerImpl implements AMContainer {
private final StateMachine<AMContainerState, AMContainerEventType, AMContainerEvent> stateMachine;
private static final StateMachineFactory
- <AMContainerImpl, AMContainerState, AMContainerEventType, AMContainerEvent>
- stateMachineFactory =
+ <AMContainerImpl, AMContainerState, AMContainerEventType, AMContainerEvent>
+ stateMachineFactory =
new StateMachineFactory<AMContainerImpl, AMContainerState, AMContainerEventType, AMContainerEvent>(
AMContainerState.ALLOCATED)
@@ -144,7 +144,7 @@ public class AMContainerImpl implements AMContainer {
.addTransition(AMContainerState.IDLE, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_TIMED_OUT, new TimedOutAtIdleTransition())
.addTransition(AMContainerState.IDLE, AMContainerState.STOPPING, AMContainerEventType.C_NODE_FAILED, new NodeFailedAtIdleTransition())
.addTransition(AMContainerState.IDLE, AMContainerState.STOP_REQUESTED, EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, AMContainerEventType.C_LAUNCHED, AMContainerEventType.C_LAUNCH_FAILED, AMContainerEventType.C_TA_SUCCEEDED, AMContainerEventType.C_NM_STOP_SENT, AMContainerEventType.C_NM_STOP_FAILED), new ErrorAtIdleTransition())
-
+
.addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_ASSIGN_TA, new AssignTaskAttemptAtRunningTransition())
.addTransition(AMContainerState.RUNNING, AMContainerState.RUNNING, AMContainerEventType.C_PULL_TA)
.addTransition(AMContainerState.RUNNING, AMContainerState.IDLE, AMContainerEventType.C_TA_SUCCEEDED, new TASucceededAtRunningTransition())
@@ -162,7 +162,7 @@ public class AMContainerImpl implements AMContainer {
.addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.STOPPING, AMContainerEventType.C_NODE_FAILED, new NodeFailedAtNMStopRequestedTransition())
.addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.STOP_REQUESTED, EnumSet.of(AMContainerEventType.C_LAUNCHED, AMContainerEventType.C_LAUNCH_FAILED, AMContainerEventType.C_TA_SUCCEEDED, AMContainerEventType.C_STOP_REQUEST, AMContainerEventType.C_TIMED_OUT))
.addTransition(AMContainerState.STOP_REQUESTED, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_LAUNCH_REQUEST, new ErrorAtNMStopRequestedTransition())
-
+
.addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_ASSIGN_TA, new AssignTAAtWindDownTransition())
.addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, AMContainerEventType.C_PULL_TA, new PullTAAfterStopTransition())
// TODO This transition is wrong. Should be a noop / error.
@@ -196,10 +196,10 @@ public class AMContainerImpl implements AMContainer {
this.taskAttemptListener = tal;
this.failedAssignments = new LinkedList<TezTaskAttemptID>();
- this.noAllocationContainerTask = WAIT_TASK;
+ this.noAllocationContainerTask = WAIT_TASK;
this.stateMachine = stateMachineFactory.make(this);
}
-
+
@Override
public AMContainerState getState() {
readLock.lock();
@@ -214,7 +214,7 @@ public class AMContainerImpl implements AMContainer {
public ContainerId getContainerId() {
return this.containerId;
}
-
+
@Override
public Container getContainer() {
return this.container;
@@ -262,7 +262,7 @@ public class AMContainerImpl implements AMContainer {
readLock.unlock();
}
}
-
+
@Override
public int getShufflePort() {
readLock.lock();
@@ -272,17 +272,19 @@ public class AMContainerImpl implements AMContainer {
readLock.unlock();
}
}
-
+
public boolean isInErrorState() {
return inError;
}
@Override
public void handle(AMContainerEvent event) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing AMContainerEvent " + event.getContainerId()
+ + " of type " + event.getType() + " while in state: " + getState()
+ + ". Event: " + event);
+ }
this.writeLock.lock();
- LOG.info("DEBUG: Processing AMContainerEvent " + event.getContainerId()
- + " of type " + event.getType() + " while in state: " + getState()
- + ". Event: " + event);
try {
final AMContainerState oldState = getState();
try {
@@ -302,7 +304,7 @@ public class AMContainerImpl implements AMContainer {
writeLock.unlock();
}
}
-
+
@SuppressWarnings("unchecked")
private void sendEvent(Event<?> event) {
this.eventHandler.handle(event);
@@ -348,14 +350,14 @@ public class AMContainerImpl implements AMContainer {
container.taskAttemptListener, event.getCredentials(),
event.shouldProfile(), container.appContext);
- // Registering now, so that in case of delayed NM response, the child
+ // Registering now, so that in case of delayed NM response, the child
// task is not told to die since the TAL does not know about the container.
container.registerWithTAListener();
container.sendStartRequestToNM();
LOG.info("Sending Launch Request for Container with id: " +
container.container.getId());
// Forget about the clc to save resources. At some point, part of the clc
- // info may need to be exposed to the scheduler to figure out whether a
+ // info may need to be exposed to the scheduler to figure out whether a
// container can be used for a specific TaskAttempt.
container.clc = null;
}
@@ -451,7 +453,9 @@ public class AMContainerImpl implements AMContainer {
return AMContainerState.STOP_REQUESTED;
}
container.pendingAttempt = event.getTaskAttemptId();
- LOG.info("DEBUG: AssignTA: attempt: " + event.getRemoteTaskContext());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AssignTA: attempt: " + event.getRemoteTaskContext());
+ }
container.remoteTaskMap
.put(event.getTaskAttemptId(), event.getRemoteTaskContext());
return container.getState();
@@ -580,7 +584,7 @@ public class AMContainerImpl implements AMContainer {
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
super.transition(container, cEvent);
if (container.pendingAttempt != null) {
- container.sendTerminatingToTaskAttempt(container.pendingAttempt,
+ container.sendTerminatingToTaskAttempt(container.pendingAttempt,
"Container " + container.getContainerId() +
" hit an invalid transition - " + cEvent.getType() + " at " +
container.getState());
@@ -595,8 +599,10 @@ public class AMContainerImpl implements AMContainer {
@Override
public AMContainerState transition(
AMContainerImpl container, AMContainerEvent cEvent) {
- LOG.info("DEBUG: AssignTAAtIdle: attempt: " +
- ((AMContainerEventAssignTA) cEvent).getRemoteTaskContext());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AssignTAAtIdle: attempt: " +
+ ((AMContainerEventAssignTA) cEvent).getRemoteTaskContext());
+ }
return super.transition(container, cEvent);
}
}
@@ -617,12 +623,13 @@ public class AMContainerImpl implements AMContainer {
if (container.lastTaskFinishTime != 0) {
long idleTimeDiff =
System.currentTimeMillis() - container.lastTaskFinishTime;
- LOG.info("DEBUG: Computing idle time for container: " +
- container.getContainerId() + ", lastFinishTime: " +
- container.lastTaskFinishTime + ", Incremented by: " +
- idleTimeDiff);
- container.idleTimeBetweenTasks +=
- System.currentTimeMillis() - container.lastTaskFinishTime;
+ container.idleTimeBetweenTasks += idleTimeDiff;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Computing idle time for container: " +
+ container.getContainerId() + ", lastFinishTime: " +
+ container.lastTaskFinishTime + ", Incremented by: " +
+ idleTimeDiff);
+ }
}
LOG.info("Assigned taskAttempt + [" + container.runningAttempt +
"] to container: [" + container.getContainerId() + "]");
@@ -656,7 +663,9 @@ public class AMContainerImpl implements AMContainer {
@Override
public void transition(AMContainerImpl container, AMContainerEvent cEvent) {
super.transition(container, cEvent);
- LOG.info("DEBUG: IdleTimeBetweenTasks: " + container.idleTimeBetweenTasks);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("IdleTimeBetweenTasks: " + container.idleTimeBetweenTasks);
+ }
container.unregisterFromContainerListener();
}
}
@@ -785,7 +794,7 @@ public class AMContainerImpl implements AMContainer {
container.registerFailedAttempt(event.getTaskAttemptId());
}
}
-
+
// Hack to some extent. This allocation should be done while entering one of
// the post-running states, insetad of being a transition on the post stop
// states.
@@ -907,11 +916,11 @@ public class AMContainerImpl implements AMContainer {
protected void registerFailedAttempt(TezTaskAttemptID taId) {
failedAssignments.add(taId);
}
-
+
protected void deAllocate() {
sendEvent(new AMSchedulerEventDeallocateContainer(containerId));
}
-
+
protected void sendCompletedToScheduler() {
sendEvent(new AMSchedulerEventContainerCompleted(containerId));
}
@@ -944,7 +953,7 @@ public class AMContainerImpl implements AMContainer {
sendEvent(new NMCommunicatorStopRequestEvent(containerId,
container.getNodeId(), container.getContainerToken()));
}
-
+
protected void unregisterAttemptFromListener(TezTaskAttemptID attemptId) {
taskAttemptListener.unregisterTaskAttempt(attemptId);
}
@@ -965,7 +974,7 @@ public class AMContainerImpl implements AMContainer {
protected void unregisterFromContainerListener() {
this.containerHeartbeatHandler.unregister(this.containerId);
}
-
-
+
+
}
[03/50] [abbrv] git commit: TEZ-25: Change DAG representation from
Configuration object to structured protobuf message
Posted by ss...@apache.org.
TEZ-25: Change DAG representation from Configuration object to structured protobuf message
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/cb5758b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/cb5758b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/cb5758b4
Branch: refs/heads/master
Commit: cb5758b424a80c5a17e0d9983a7b6ff2982dcd31
Parents: 97e1fa9
Author: mikelid <mi...@microsoft.com>
Authored: Mon May 13 11:21:27 2013 -0700
Committer: mikelid <mi...@microsoft.com>
Committed: Mon May 13 11:21:27 2013 -0700
----------------------------------------------------------------------
pom.xml | 5 +
tez-dag-api/pom.xml | 26 +
.../src/main/java/org/apache/tez/dag/api/DAG.java | 128 ++++-
.../org/apache/tez/dag/api/DAGConfiguration.java | 492 ---------------
.../org/apache/tez/dag/api/DAGLocationHint.java | 95 ---
.../org/apache/tez/dag/api/DagTypeConverters.java | 221 +++++++
.../org/apache/tez/dag/api/TezConfiguration.java | 3 +-
.../org/apache/tez/dag/api/VertexLocationHint.java | 83 +---
tez-dag-api/src/main/proto/DAGPlan.proto | 119 ++++
.../java/org/apache/tez/dag/api/TestDAGPlan.java | 82 +++
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 82 ++--
.../org/apache/tez/dag/app/MRRExampleHelper.java | 33 +-
.../main/java/org/apache/tez/dag/app/dag/DAG.java | 7 +-
.../java/org/apache/tez/dag/app/dag/Vertex.java | 3 +
.../org/apache/tez/dag/app/dag/impl/DAGImpl.java | 101 ++--
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 33 +-
.../apache/tez/dag/utils/TezEngineChildJVM.java | 6 +
.../apache/tez/dag/api/TestDAGLocationHint.java | 114 ----
.../apache/tez/dag/api/TestVertexLocationHint.java | 147 -----
.../java/org/apache/tez/mapreduce/YARNRunner.java | 50 +-
20 files changed, 736 insertions(+), 1094 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 586eb18..5c545c5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -248,6 +248,11 @@
<artifactId>avro</artifactId>
<version>1.5.3</version>
</dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>2.4.0a</version>
+ </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag-api/pom.xml b/tez-dag-api/pom.xml
index c084f01..2b6aebb 100644
--- a/tez-dag-api/pom.xml
+++ b/tez-dag-api/pom.xml
@@ -46,6 +46,32 @@
<configuration>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <version>3.0.0-SNAPSHOT</version>
+ <executions>
+ <execution>
+ <id>compile-protoc</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <imports>
+ <param>${basedir}/src/main/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/main/proto</directory>
+ <includes>
+ <include>DAGPlan.proto</include>
+ </includes>
+ </source>
+ <output>${project.build.directory}/generated-sources/java</output>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 185f9a4..c18c720 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -18,8 +18,24 @@
package org.apache.tez.dag.api;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.tez.dag.api.*;
+import org.apache.tez.dag.api.DAGPlan.PlanKeyValuePair.Builder;
+import org.apache.tez.dag.api.DAGPlan.PlanTaskLocationHint;
+import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
+import org.apache.tez.dag.api.EdgeProperty.SourceType;
+import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
+import org.apache.tez.dag.api.DAGPlan.*;
+
public class DAG { // FIXME rename to Topology
List<Vertex> vertices;
@@ -76,28 +92,102 @@ public class DAG { // FIXME rename to Topology
//for each vertex if not specified?
}
- // FIXME DAGConfiguration is not public API
- public DAGConfiguration serializeDag() {
- DAGConfiguration dagConf = new DAGConfiguration();
-
- dagConf.setName(name);
- dagConf.setVertices(vertices);
- dagConf.setEdgeProperties(edges);
-
- for(Vertex vertex : vertices) {
- if(vertex.getInputVertices() != null) {
- dagConf.setInputVertices(vertex.getVertexName(), vertex.getInputVertices());
- dagConf.setInputEdgeIds(vertex.getVertexName(), vertex.getInputEdgeIds());
+ // create protobuf message describing DAG
+ public JobPlan createDag(){
+ JobPlan.Builder jobBuilder = JobPlan.newBuilder();
+
+ jobBuilder.setName(this.name);
+
+ for(Vertex vertex : vertices){
+ VertexPlan.Builder vertexBuilder = VertexPlan.newBuilder();
+ vertexBuilder.setName(vertex.getVertexName());
+ vertexBuilder.setType(PlanVertexType.NORMAL); // vertex type is implicitly NORMAL until TEZ-46.
+ vertexBuilder.setProcessorName(vertex.getProcessorName());
+
+ //task config
+ PlanTaskConfiguration.Builder taskConfigBuilder = PlanTaskConfiguration.newBuilder();
+ Resource resource = vertex.getTaskResource();
+ taskConfigBuilder.setNumTasks(vertex.getParallelism());
+ taskConfigBuilder.setMemoryMb(resource.getMemory());
+ taskConfigBuilder.setVirtualCores(resource.getVirtualCores());
+ taskConfigBuilder.setJavaOpts(vertex.getJavaOpts());
+
+ taskConfigBuilder.setTaskModule(vertex.getVertexName());
+ PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder();
+ Map<String,LocalResource> lrs = vertex.getTaskLocalResources();
+ for(String key : lrs.keySet()){
+ LocalResource lr = lrs.get(key);
+ localResourcesBuilder.setName(key);
+ localResourcesBuilder.setUri(DagTypeConverters.convertToDAGPlan(lr.getResource()));
+ localResourcesBuilder.setSize(lr.getSize());
+ localResourcesBuilder.setTimeStamp(lr.getTimestamp());
+ localResourcesBuilder.setType(DagTypeConverters.convertToDAGPlan(lr.getType()));
+ localResourcesBuilder.setVisibility(DagTypeConverters.convertToDAGPlan(lr.getVisibility()));
+ if(lr.getType() == LocalResourceType.PATTERN){
+ assert lr.getPattern() != null : "resourceType=PATTERN but pattern is null";
+ localResourcesBuilder.setPattern(lr.getPattern());
+ }
+ taskConfigBuilder.addLocalResource(localResourcesBuilder);
}
- if(vertex.getOutputVertices() != null) {
- dagConf.setOutputVertices(vertex.getVertexName(), vertex.getOutputVertices());
- dagConf.setOutputEdgeIds(vertex.getVertexName(), vertex.getOutputEdgeIds());
+
+ if(vertex.getTaskEnvironment() != null){
+ for(String key : vertex.getTaskEnvironment().keySet()){
+ PlanKeyValuePair.Builder envSettingBuilder = PlanKeyValuePair.newBuilder();
+ envSettingBuilder.setKey(key);
+ envSettingBuilder.setValue(vertex.getTaskEnvironment().get(key));
+ taskConfigBuilder.addEnvironmentSetting(envSettingBuilder);
+ }
}
+
+ if(vertex.getTaskLocationsHint() != null ){
+ if(vertex.getTaskLocationsHint().getTaskLocationHints() != null){
+ for(TaskLocationHint hint : vertex.getTaskLocationsHint().getTaskLocationHints()){
+ PlanTaskLocationHint.Builder taskLocationHintBuilder = PlanTaskLocationHint.newBuilder();
+
+ if(hint.getDataLocalHosts() != null){
+ taskLocationHintBuilder.addAllHost(Arrays.asList(hint.getDataLocalHosts()));
+ }
+ if(hint.getRacks() != null){
+ taskLocationHintBuilder.addAllRack(Arrays.asList(hint.getRacks()));
+ }
+
+ vertexBuilder.addTaskLocationHint(taskLocationHintBuilder);
+ }
+ }
+ }
+
+ for(String inEdgeId : vertex.getInputEdgeIds()){
+ vertexBuilder.addInEdgeId(inEdgeId);
+ }
+
+ for(String outEdgeId : vertex.getOutputEdgeIds()){
+ vertexBuilder.addOutEdgeId(outEdgeId);
+ }
+
+ vertexBuilder.setTaskConfig(taskConfigBuilder);
+ jobBuilder.addVertex(vertexBuilder);
+ }
+
+ for(Edge edge : edges){
+ EdgePlan.Builder edgeBuilder = EdgePlan.newBuilder();
+ edgeBuilder.setId(edge.getId());
+ edgeBuilder.setInputVertexName(edge.getInputVertex().getVertexName());
+ edgeBuilder.setOutputVertexName(edge.getOutputVertex().getVertexName());
+ edgeBuilder.setConnectionPattern(DagTypeConverters.convertToDAGPlan(edge.getEdgeProperty().connectionPattern));
+ edgeBuilder.setSourceType(DagTypeConverters.convertToDAGPlan(edge.getEdgeProperty().getSourceType()));
+ edgeBuilder.setInputClass(edge.getEdgeProperty().inputClass);
+ edgeBuilder.setOutputClass(edge.getEdgeProperty().outputClass);
+
+ jobBuilder.addEdge(edgeBuilder);
}
- dagConf.setConfig(config);
-
- return dagConf;
+ for(Entry<String, String> entry : this.config.entrySet()){
+ PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
+ kvp.setKey(entry.getKey());
+ kvp.setValue(entry.getValue());
+ jobBuilder.addJobSetting(kvp);
+ }
+
+ return jobBuilder.build();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
deleted file mode 100644
index 93ed118..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
+++ /dev/null
@@ -1,492 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api;
-
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.StringTokenizer;
-import java.util.TreeMap;
-import java.util.Map.Entry;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
-
-// FIXME dag conf should not be public API???
-public class DAGConfiguration extends Configuration {
-
- private static final Log LOG = LogFactory.getLog(DAGConfiguration.class);
-
- public DAGConfiguration(Configuration conf) {
- super(conf);
- if (! (conf instanceof DAGConfiguration)) {
- this.reloadConfiguration();
- }
- }
-
- public DAGConfiguration() {
- super();
- }
-
- public final static String DAG = "tez.dag.";
-
- public final static String DAG_AM = DAG + "am.";
-
- public final static String VERTEX = DAG + "vertex.";
-
- public final static String TASK = DAG + "task.";
-
- public final static String TASK_ATTEMPT = DAG + "attempt.";
-
- public final static String TEZ_DAG_VERTICES = DAG + "vertices";
-
- public final static String TEZ_DAG_EDGES = DAG + "edges";
-
- public final static String EDGE = DAG + "edge.";
-
- private final static String SEPARATOR = "|";
-
- public final static String TEZ_DAG_CONFIG_KEYS = DAG + "keys";
- public final static String TEZ_DAG_CONFIG_VALUES = DAG + "values";
-
- @Private
- public void setConfig(Map<String, String> config) {
- if(!config.isEmpty()) {
- String[] key = new String[config.size()];
- String[] value = new String[config.size()];
- int i=0;
- for(Entry<String, String> entry : config.entrySet()) {
- key[i] = entry.getKey();
- value[i] = entry.getValue();
- i++;
- }
- setStrings(TEZ_DAG_CONFIG_KEYS, key);
- setStrings(TEZ_DAG_CONFIG_VALUES, value);
- }
- }
-
- @Private
- public Map<String, String> getConfig() {
- HashMap<String, String> config = new HashMap<String, String>();
- String[] key = getStrings(TEZ_DAG_CONFIG_KEYS);
- if(key != null) {
- String[] value = getStrings(TEZ_DAG_CONFIG_VALUES);
- assert value.length == key.length;
- for(int i=0; i<key.length; ++i) {
- config.put(key[i], value[i]);
- }
- }
- return config;
- }
-
- @Private
- public void setEdgeProperties(List<Edge> edges) {
- String[] edgeIds = new String[edges.size()];
- for(int i=0; i<edges.size(); ++i) {
- edgeIds[i] = edges.get(i).getId();
- }
- setStrings(TEZ_DAG_EDGES, edgeIds);
- for (Edge edge : edges) {
- setEdgeProperty(edge.getEdgeProperty(), edge.getId());
- }
- }
-
- public Map<String, EdgeProperty> getEdgeProperties() {
- String[] edgeIds = getStrings(TEZ_DAG_EDGES);
- if (edgeIds == null) {
- return new TreeMap<String, EdgeProperty>();
- }
- Map<String, EdgeProperty> edgeProperties =
- new HashMap<String, EdgeProperty>(edgeIds.length);
- for(int i=0; i<edgeIds.length; ++i) {
- edgeProperties.put(edgeIds[i], getEdgeProperty(edgeIds[i]));
- }
- return edgeProperties;
- }
-
- private void setEdgeProperty(EdgeProperty edgeProperty, String edgeId) {
- String[] edgeStrs = new String[4];
- edgeStrs[0] = edgeProperty.getConnectionPattern().name();
- edgeStrs[1] = edgeProperty.getSourceType().name();
- edgeStrs[2] = edgeProperty.inputClass;
- edgeStrs[3] = edgeProperty.outputClass;
-
- setStrings(EDGE + edgeId, edgeStrs);
- }
-
- private EdgeProperty getEdgeProperty(String edgeId) {
- String[] edgeStr = getStrings(EDGE + edgeId);
- assert edgeStr.length == 4;
- return new EdgeProperty(EdgeProperty.ConnectionPattern.valueOf(edgeStr[0]),
- EdgeProperty.SourceType.valueOf(edgeStr[1]),
- edgeStr[2],
- edgeStr[3]);
- }
-
- public final static String TEZ_DAG_VERTEX_TASKS = VERTEX + "num-tasks";
- public final static int DEFAULT_TEZ_DAG_VERTEX_TASKS = 0;
-
- public int getNumVertexTasks(String vertexName) {
- return getInt(
- TEZ_DAG_VERTEX_TASKS + "." + vertexName,
- DEFAULT_TEZ_DAG_VERTEX_TASKS);
- }
-
- public void setNumVertexTasks(String vertexName, int numTasks) {
- setInt(
- TEZ_DAG_VERTEX_TASKS + "." + vertexName,
- numTasks);
- }
-
- public final String TEZ_DAG_VERTEX_TASK_MEMORY = TASK + "memory-mb";
-
- public final int DEFAULT_TEZ_DAG_VERTEX_TASK_MEMORY = 1024;
-
- public int getVertexTaskMemory(String vertexName) {
- return getInt(
- TEZ_DAG_VERTEX_TASK_MEMORY + "." + vertexName,
- DEFAULT_TEZ_DAG_VERTEX_TASK_MEMORY);
- }
-
- public void setVertexTaskMemory(String vertexName, int memory) {
- setInt(
- TEZ_DAG_VERTEX_TASK_MEMORY + "." + vertexName,
- memory);
- }
-
- public final String TEZ_DAG_VERTEX_TASK_CPU = TASK + "cpu-vcores";
-
- public final int DEFAULT_TEZ_DAG_VERTEX_TASK_CORES = 1;
-
- public int getVertexTaskCores(String vertexName) {
- return getInt(
- TEZ_DAG_VERTEX_TASK_CPU + "." + vertexName,
- DEFAULT_TEZ_DAG_VERTEX_TASK_CORES);
- }
- public void setVertexTaskCores(String vertexName, int cores) {
- setInt(
- TEZ_DAG_VERTEX_TASK_CPU + "." + vertexName,
- cores);
- }
-
- private final String[] EMPTY = new String[0];
-
- public String[] getVertices() {
- String[] vertices = getStrings(TEZ_DAG_VERTICES, EMPTY);
- return vertices == null? EMPTY : vertices;
- }
-
- void setVertexResource(Vertex vertex) {
- Resource resource = vertex.getTaskResource();
- if (resource == null) {
- return;
- }
- setVertexTaskCores(vertex.getVertexName(), resource.getVirtualCores());
- setVertexTaskMemory(vertex.getVertexName(), resource.getMemory());
- }
-
- public Resource getVertexResource(String vertexName) {
- int memory = getVertexTaskMemory(vertexName);
- int vCores = getVertexTaskCores(vertexName);
- return BuilderUtils.newResource(memory, vCores);
- }
-
- // FIXME we are serializing YarnURL which is not same as serializing a URL
- public final String TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE = TASK + "local-resource.";
- void setVertexLocalResource(Vertex vertex) {
- Map<String,LocalResource> lrs = vertex.getTaskLocalResources();
- if (lrs == null) {
- return;
- }
- String[] lrStrs = new String[lrs.size()];
- int i=0;
- for(Map.Entry<String,LocalResource> entry : lrs.entrySet()) {
- LocalResource lr = entry.getValue();
- try {
- String lrStr = StringUtils.escapeString(entry.getKey(),
- StringUtils.ESCAPE_CHAR,
- SEPARATOR.charAt(0))
- + SEPARATOR
- + StringUtils.escapeString(
- ConverterUtils.getPathFromYarnURL(lr.getResource()).toString(),
- StringUtils.ESCAPE_CHAR,
- SEPARATOR.charAt(0))
- + SEPARATOR
- + StringUtils.escapeString(String.valueOf(lr.getSize()),
- StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
- + SEPARATOR
- + StringUtils.escapeString(String.valueOf(lr.getTimestamp()),
- StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
- + SEPARATOR
- + StringUtils.escapeString(lr.getType().name(),
- StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
- + SEPARATOR
- + StringUtils.escapeString(lr.getVisibility().name(),
- StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
- + SEPARATOR
- + StringUtils.escapeString(
- (lr.getPattern() == null ? "" : lr.getPattern()),
- StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
- lrStrs[i++] = StringUtils.escapeString(lrStr);
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
- }
- setStrings(TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE + vertex.getVertexName(),
- lrStrs);
- }
-
- public Map<String, LocalResource> getVertexLocalResources(String vertexName) {
- String[] lrStrs = StringUtils.split(get(
- TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE + vertexName, ""));
- Map<String, LocalResource> localResources =
- new TreeMap<String, LocalResource>();
- if (lrStrs == null) {
- return localResources;
- }
- LOG.info("XXXX Found " + lrStrs.length + " local resources");
- for (String lrStr : lrStrs) {
- String[] tokens =
- StringUtils.split(
- lrStr, StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
- if (tokens.length != 6
- && tokens.length != 7) {
- LOG.warn("Invalid token count in serialized LocalResource"
- + ", serializedString=" + lrStr
- + ", tokenCount=" + tokens.length);
- }
- String resourceName = tokens[0];
- LocalResource lRsrc = Records.newRecord(LocalResource.class);
- lRsrc.setResource(ConverterUtils.getYarnUrlFromPath(
- new Path(tokens[1])));
- lRsrc.setSize(Long.valueOf(tokens[2]));
- lRsrc.setTimestamp(Long.valueOf(tokens[3]));
- lRsrc.setType(LocalResourceType.valueOf(tokens[4]));
- lRsrc.setVisibility(LocalResourceVisibility.valueOf(
- tokens[5]));
- if (tokens.length == 7) {
- lRsrc.setPattern(tokens[6]);
- }
- try {
- LOG.info("XXXX Adding local resource"
- + ", vertexName=" + vertexName
- + ", resourceName=" + resourceName
- + ", resourceUrl"
- + ConverterUtils.getPathFromYarnURL(
- lRsrc.getResource()).toString());
- } catch (URISyntaxException e) {
- // Ignore
- // FIXME
- }
- localResources.put(resourceName, lRsrc);
- }
- return localResources;
- }
-
- public final String TEZ_DAG_VERTEX_TASK_ENV = TASK + "env.";
- void setVertexEnv(Vertex vertex) {
- Map<String, String> env = vertex.getTaskEnvironment();
- if (env == null) {
- return;
- }
- String[] envStrs = new String[env.size()];
- int i=0;
- for(Map.Entry<String,String> entry : env.entrySet()) {
- String envStr = entry.getKey() + SEPARATOR + entry.getValue();
- envStrs[i++] = StringUtils.escapeString(envStr);
- }
- set(TEZ_DAG_VERTEX_TASK_ENV + vertex.getVertexName(),
- StringUtils.join(",", envStrs));
- }
-
- public Map<String,String> getVertexEnv(String vertexName) {
- String[] envStrs = StringUtils.split(
- get(TEZ_DAG_VERTEX_TASK_ENV + vertexName, ""));
- Map<String,String> env = new HashMap<String,String>();
- if(envStrs == null) {
- return env;
- }
-
- LOG.info("XXXX Found " + envStrs.length + " environment");
- for(String envStr : envStrs ) {
- LOG.info("XXXX Parsing env from " + envStr);
- StringTokenizer tokenizer = new StringTokenizer (envStr, SEPARATOR);
- String envName = tokenizer.nextToken();
- String envValue = tokenizer.nextToken();
- env.put(StringUtils.unEscapeString(envName),
- StringUtils.unEscapeString(envValue));
- }
-
- return env;
- }
-
- public final String TEZ_DAG_NAME = DAG + "name";
- @Private
- public void setName(String name) {
- set(TEZ_DAG_NAME, name);
- }
-
- @Public
- @Stable
- public String getName() {
- String name = get(TEZ_DAG_NAME);
- return name;
- }
-
- @Private
- public void setVertices(List<Vertex> vertices) {
- setVertices(TEZ_DAG_VERTICES, vertices);
- for(Vertex vertex : vertices) {
- // set num tasks
- setNumVertexTasks(vertex.getVertexName(), vertex.getParallelism());
- // set resource
- setVertexResource(vertex);
- // set localResource
- setVertexLocalResource(vertex);
- // set environment
- setVertexEnv(vertex);
- // set processor name
- setVertexTaskModuleClassName(vertex);
- //set javaOpts
- setVertexJavaOpts(vertex.getVertexName(), vertex.getJavaOpts());
- }
- }
-
- public final String TEZ_DAG_VERTEX_INPUT_VERTICES = VERTEX + "input-vertices";
- @Public
- @Stable
- public String[] getInputVertices(String vertexName) {
- String[] vertices =
- getStrings(TEZ_DAG_VERTEX_INPUT_VERTICES + "." + vertexName, EMPTY);
- return vertices == null ? EMPTY : vertices;
- }
- @Private
- public void setInputVertices(String vertexName, List<Vertex> inputVertices) {
- setVertices(TEZ_DAG_VERTEX_INPUT_VERTICES + "." + vertexName,
- inputVertices);
- }
-
- private void setVertices(String key, List<Vertex> vertices) {
- String[] verticesNames = new String[vertices.size()];
- for (int i = 0; i < vertices.size(); ++i) {
- verticesNames[i] = vertices.get(i).getVertexName();
- }
- setStrings(key, verticesNames);
- }
-
- public final String TEZ_DAG_VERTEX_OUTPUT_VERTICES = VERTEX
- + "output-vertices";
- @Public
- @Stable
- public String[] getOutputVertices(String vertexName) {
- String[] vertices =
- getStrings(TEZ_DAG_VERTEX_OUTPUT_VERTICES + "." + vertexName, EMPTY);
- return vertices == null? EMPTY : vertices;
- }
- @Private
- public void setOutputVertices(String vertexName,
- List<Vertex> outputVertices) {
- setVertices(TEZ_DAG_VERTEX_OUTPUT_VERTICES + "." + vertexName,
- outputVertices);
- }
-
- public final String TEZ_DAG_VERTEX_INPUT_EDGES = VERTEX + "input-edges";
- @Public
- @Stable
- public List<String> getInputEdgeIds(String vertexName) {
- return getEdgeIds(TEZ_DAG_VERTEX_INPUT_EDGES + "." + vertexName);
- }
- public void setInputEdgeIds(String vertexName, List<String> edgeIds) {
- setEdgeIds(TEZ_DAG_VERTEX_INPUT_EDGES + "." + vertexName, edgeIds);
- }
-
- public final String TEZ_DAG_VERTEX_OUTPUT_EDGES = VERTEX + "output-edges";
- @Public
- @Stable
- public List<String> getOutputEdgeIds(String vertexName) {
- return getEdgeIds(TEZ_DAG_VERTEX_OUTPUT_EDGES + "." + vertexName);
- }
- @Private
- public void setOutputEdgeIds(String vertexName, List<String> edgeIds) {
- setEdgeIds(TEZ_DAG_VERTEX_OUTPUT_EDGES + "." + vertexName, edgeIds);
- }
-
- private List<String> getEdgeIds(String key) {
- String[] edgeIds = getStrings(key, EMPTY);
- if (edgeIds == null) {
- return new ArrayList<String>();
- }
- return Arrays.asList(edgeIds);
- }
-
- private void setEdgeIds(String key, List<String> edgeIds) {
- setStrings(key, edgeIds.toArray(new String[]{}));
- }
-
- private void setVertexTaskModuleClassName(Vertex vertex) {
- setVertexTaskModuleClassName(vertex.getVertexName(),
- vertex.getProcessorName());
- }
-
- public final String TEZ_DAG_VERTEX_TASK_MODULE= VERTEX + "task-module";
- @Private
- public String getVertexTaskModuleClassName(String vertexName) {
- return get(TEZ_DAG_VERTEX_TASK_MODULE + "." + vertexName);
- }
- @Private
- public void setVertexTaskModuleClassName(String vertexName,
- String taskModule) {
- set(TEZ_DAG_VERTEX_TASK_MODULE + "." + vertexName, taskModule);
- }
-
- public final String TEZ_DAG_VERTEX_JAVAOPTS= VERTEX + "java-opts";
- @Private
- public String getVertexJavaOpts(String vertexName) {
- String opts = get(TEZ_DAG_VERTEX_JAVAOPTS + "." + vertexName);
- return opts == null? "" : opts;
- }
-
- @Private
- public void setVertexJavaOpts(String vertexName, String javaOpts){
- set(TEZ_DAG_VERTEX_JAVAOPTS + "." + vertexName, javaOpts);
- }
-
- /// File used for storing location hints that are passed to the DAG
- public static final String DAG_LOCATION_HINT_RESOURCE_FILE =
- DAG + "location-hint-resource-file";
- public static final String DEFAULT_DAG_LOCATION_HINT_RESOURCE_FILE =
- "tezdaglocationhint.info";
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java
deleted file mode 100644
index 9f48f2c..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGLocationHint.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api;
-
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeMap;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-
-public class DAGLocationHint implements Writable {
-
- private Map<String, VertexLocationHint> vertexLocationHints;
-
- public DAGLocationHint() {
- vertexLocationHints = new TreeMap<String, VertexLocationHint>();
- }
-
- public Map<String, VertexLocationHint> getVertexLocationHints() {
- return vertexLocationHints;
- }
-
- public VertexLocationHint getVertexLocationHint(String vertexName) {
- return vertexLocationHints.get(vertexName);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(vertexLocationHints.size());
- for (Entry<String, VertexLocationHint> entry :
- vertexLocationHints.entrySet()) {
- Text.writeString(out, entry.getKey());
- out.writeBoolean(entry.getValue() != null);
- if (entry.getValue() != null) {
- entry.getValue().write(out);
- }
- }
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- int entryCount = in.readInt();
- vertexLocationHints = new TreeMap<String, VertexLocationHint>();
- for (int i = 0; i < entryCount; ++i) {
- String vertexName = Text.readString(in);
- if (!in.readBoolean()) {
- vertexLocationHints.put(vertexName, null);
- } else {
- VertexLocationHint hint = new VertexLocationHint();
- hint.readFields(in);
- vertexLocationHints.put(vertexName, hint);
- }
- }
- }
-
- public static DAGLocationHint initDAGDagLocationHint(
- String locationHintFile) throws IOException {
- DataInput in = new DataInputStream(new FileInputStream(locationHintFile));
- DAGLocationHint dagLocationHint = new DAGLocationHint();
- dagLocationHint.readFields(in);
- return dagLocationHint;
- }
-
- public static void writeDAGDagLocationHint(
- DAGLocationHint dagLocationHint,
- String locationHintFile) throws IOException {
- DataOutput out = new DataOutputStream(new FileOutputStream(
- locationHintFile));
- dagLocationHint.write(out);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
new file mode 100644
index 0000000..04404ba
--- /dev/null
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.dag.api;
+
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.dag.api.DAGPlan.EdgePlan;
+import org.apache.tez.dag.api.DAGPlan.PlanKeyValuePair;
+import org.apache.tez.dag.api.DAGPlan.PlanLocalResource;
+import org.apache.tez.dag.api.DAGPlan.PlanTaskConfiguration;
+import org.apache.tez.dag.api.DAGPlan.PlanTaskLocationHint;
+import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
+import org.apache.tez.dag.api.EdgeProperty.SourceType;
+import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
+import org.apache.tez.dag.api.DAGPlan.*;
+
+public class DagTypeConverters {
+
+ public static PlanLocalResourceVisibility convertToDAGPlan(LocalResourceVisibility visibility){
+ switch(visibility){
+ case PUBLIC : return PlanLocalResourceVisibility.PUBLIC;
+ case PRIVATE : return PlanLocalResourceVisibility.PRIVATE;
+ case APPLICATION : return PlanLocalResourceVisibility.APPLICATION;
+ default : throw new RuntimeException("unknown 'visibility'");
+ }
+ }
+
+ public static LocalResourceVisibility convertFromDAGPlan(PlanLocalResourceVisibility visibility){
+ switch(visibility){
+ case PUBLIC : return LocalResourceVisibility.PUBLIC;
+ case PRIVATE : return LocalResourceVisibility.PRIVATE;
+ case APPLICATION : return LocalResourceVisibility.APPLICATION;
+ default : throw new RuntimeException("unknown 'visibility'");
+ }
+ }
+
+ public static PlanEdgeSourceType convertToDAGPlan(SourceType sourceType){
+ switch(sourceType){
+ case STABLE : return PlanEdgeSourceType.STABLE;
+ case STABLE_PERSISTED : return PlanEdgeSourceType.STABLE_PERSISTED;
+ case STREAMING : return PlanEdgeSourceType.STREAMING;
+ default : throw new RuntimeException("unknown 'sourceType'");
+ }
+ }
+
+ public static SourceType convertFromDAGPlan(PlanEdgeSourceType sourceType){
+ switch(sourceType){
+ case STABLE : return SourceType.STABLE;
+ case STABLE_PERSISTED : return SourceType.STABLE_PERSISTED;
+ case STREAMING : return SourceType.STREAMING;
+ default : throw new RuntimeException("unknown 'sourceType'");
+ }
+ }
+
+ public static PlanEdgeConnectionPattern convertToDAGPlan(ConnectionPattern pattern){
+ switch(pattern){
+ case ONE_TO_ONE : return PlanEdgeConnectionPattern.ONE_TO_ONE;
+ case ONE_TO_ALL : return PlanEdgeConnectionPattern.ONE_TO_ALL;
+ case BIPARTITE : return PlanEdgeConnectionPattern.BIPARTITE;
+ default : throw new RuntimeException("unknown 'pattern'");
+ }
+ }
+
+ public static ConnectionPattern convertFromDAGPlan(PlanEdgeConnectionPattern pattern){
+ switch(pattern){
+ case ONE_TO_ONE : return ConnectionPattern.ONE_TO_ONE;
+ case ONE_TO_ALL : return ConnectionPattern.ONE_TO_ALL;
+ case BIPARTITE : return ConnectionPattern.BIPARTITE;
+ default : throw new IllegalArgumentException("unknown 'pattern'");
+ }
+ }
+
+ public static PlanLocalResourceType convertToDAGPlan(LocalResourceType type) {
+ switch(type){
+ case ARCHIVE : return PlanLocalResourceType.ARCHIVE;
+ case FILE : return PlanLocalResourceType.FILE;
+ case PATTERN : return PlanLocalResourceType.PATTERN;
+ default : throw new IllegalArgumentException("unknown 'type'");
+ }
+ }
+
+ public static LocalResourceType convertFromDAGPlan(PlanLocalResourceType type) {
+ switch(type){
+ case ARCHIVE : return LocalResourceType.ARCHIVE;
+ case FILE : return LocalResourceType.FILE;
+ case PATTERN : return LocalResourceType.PATTERN;
+ default : throw new IllegalArgumentException("unknown 'type'");
+ }
+ }
+
+ public static VertexLocationHint convertFromDAGPlan(
+ List<PlanTaskLocationHint> locationHints) {
+
+ List<TaskLocationHint> outputList = new ArrayList<TaskLocationHint>();
+
+ for(PlanTaskLocationHint inputHint : locationHints){
+ TaskLocationHint outputHint = new TaskLocationHint();
+ outputHint.setRacks(inputHint.getRackList().toArray(new String[inputHint.getRackList().size()]));
+ outputHint.setDataLocalHosts(inputHint.getHostList().toArray(new String[inputHint.getHostList().size()]));
+ outputList.add(outputHint);
+ }
+ return new VertexLocationHint(outputList.size(), outputList.toArray(new TaskLocationHint[outputList.size()]));
+ }
+
+ // notes re HDFS URL handling:
+ // Resource URLs in the protobuf message are strings of the form hdfs://host:port/path
+ // org.apache.hadoop.fs.Path.Path is actually a URI type that allows any scheme
+ // org.apache.hadoop.yarn.api.records.URL is a URL type used by YARN.
+ // java.net.URL cannot be used out of the box as it rejects unknown schemes such as HDFS.
+
+ public static String convertToDAGPlan(URL resource) {
+ // see above notes on HDFS URL handling
+ String out = resource.getScheme() + "://" + resource.getHost() + ":" + resource.getPort()
+ + resource.getFile();
+ return out;
+ }
+
+ public static Map<String, LocalResource> createLocalResourceMapFromDAGPlan(
+ List<PlanLocalResource> localResourcesList) {
+ Map<String, LocalResource> map = new HashMap<String, LocalResource>();
+ for(PlanLocalResource res : localResourcesList){
+ LocalResource r = new LocalResourcePBImpl();
+
+ //NOTE: have to check every optional field in protobuf generated classes for existence before accessing
+ //else we will receive a default value back, eg ""
+ if(res.hasPattern()){
+ r.setPattern(res.getPattern());
+ }
+ r.setResource(ConverterUtils.getYarnUrlFromPath(new Path(res.getUri()))); // see above notes on HDFS URL handling
+ r.setSize(res.getSize());
+ r.setTimestamp(res.getTimeStamp());
+ r.setType(DagTypeConverters.convertFromDAGPlan(res.getType()));
+ r.setVisibility(DagTypeConverters.convertFromDAGPlan(res.getVisibility()));
+ map.put(res.getName(), r);
+ }
+ return map;
+ }
+
+ public static Map<String, String> createEnvironmentMapFromDAGPlan(
+ List<PlanKeyValuePair> environmentSettingList) {
+
+ Map<String, String> map = new HashMap<String, String>();
+ for(PlanKeyValuePair setting : environmentSettingList){
+ map.put(setting.getKey(), setting.getValue());
+ }
+
+ return map;
+ }
+
+ public static Map<String, EdgePlan> createEdgePlanMapFromDAGPlan(List<EdgePlan> edgeList){
+ Map<String, EdgePlan> edgePlanMap =
+ new HashMap<String, EdgePlan>();
+ for(EdgePlan edgePlanItem : edgeList){
+ edgePlanMap.put(edgePlanItem.getId(), edgePlanItem);
+ }
+ return edgePlanMap;
+ }
+
+ public static Map<String, EdgeProperty> createEdgePropertyMapFromDAGPlan(
+ List<EdgePlan> edgeList) {
+
+ Map<String, EdgeProperty> map = new HashMap<String, EdgeProperty>();
+ for(EdgePlan edge: edgeList){
+ map.put(edge.getId(),
+ new EdgeProperty(
+ convertFromDAGPlan(edge.getConnectionPattern()),
+ convertFromDAGPlan(edge.getSourceType()),
+ edge.getInputClass(),
+ edge.getOutputClass()
+ )
+ );
+ }
+
+ return map;
+ }
+
+ public static Resource CreateResourceRequestFromTaskConfig(
+ PlanTaskConfiguration taskConfig) {
+ return BuilderUtils.newResource(taskConfig.getMemoryMb(), taskConfig.getVirtualCores());
+ }
+
+ public static Map<String, String> createSettingsMapFromDAGPlan(
+ List<PlanKeyValuePair> settingList) {
+ Map<String, String> map = new HashMap<String, String>();
+ for(PlanKeyValuePair setting: settingList){
+ map.put(setting.getKey(), setting.getValue());
+ }
+ return map;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index be03cd8..597fb2f 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -98,5 +98,6 @@ private static final String TEZ_CONF_DIR_ENV = "TEZ_CONF_DIR";
TEZ_HOME_ENV + "/lib/*"
};
- public static final String DAG_AM_PLAN_CONFIG_XML = "tez-dag.xml";
+ public static final String DAG_AM_PLAN_PB_BINARY = "tez-dag.pb";
+ public static final String DAG_AM_PLAN_PB_TEXT = "tez-dag.pb.txt";
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
index 8571252..381b4ba 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
@@ -18,14 +18,7 @@
package org.apache.tez.dag.api;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-
-public class VertexLocationHint implements Writable {
+public class VertexLocationHint {
private int numTasks;
private TaskLocationHint[] taskLocationHints;
@@ -56,7 +49,7 @@ public class VertexLocationHint implements Writable {
this.taskLocationHints = taskLocationHints;
}
- public static class TaskLocationHint implements Writable {
+ public static class TaskLocationHint {
// Host names if any to be used
private String[] hosts;
@@ -84,77 +77,5 @@ public class VertexLocationHint implements Writable {
public void setRacks(String[] racks) {
this.racks = racks;
}
-
- private void writeStringArray(DataOutput out, String[] array)
- throws IOException {
- if (array == null) {
- out.writeInt(-1);
- return;
- }
- out.writeInt(array.length);
- for (String entry : array) {
- out.writeBoolean(entry != null);
- if (entry != null) {
- Text.writeString(out, entry);
- }
- }
- }
-
- private String[] readStringArray(DataInput in)
- throws IOException {
- int arrayLen = in.readInt();
- if (arrayLen == -1) {
- return null;
- }
- String[] array = new String[arrayLen];
- for (int i = 0; i < arrayLen; ++i) {
- if (!in.readBoolean()) {
- array[i] = null;
- } else {
- array[i] = Text.readString(in);
- }
- }
- return array;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- writeStringArray(out, hosts);
- writeStringArray(out, racks);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- hosts = readStringArray(in);
- racks = readStringArray(in);
- }
-
-
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(numTasks);
- for (int i = 0; i < numTasks; ++i) {
- out.writeBoolean(taskLocationHints[i] != null);
- if (taskLocationHints[i] != null) {
- taskLocationHints[i].write(out);
- }
- }
}
-
- @Override
- public void readFields(DataInput in) throws IOException {
- numTasks = in.readInt();
- taskLocationHints = new TaskLocationHint[numTasks];
- for (int i = 0; i < numTasks; ++i) {
- if (!in.readBoolean()) {
- taskLocationHints[i] = null;
- } else {
- taskLocationHints[i] = new TaskLocationHint(null, null);
- taskLocationHints[i].readFields(in);
- }
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/main/proto/DAGPlan.proto
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/proto/DAGPlan.proto b/tez-dag-api/src/main/proto/DAGPlan.proto
new file mode 100644
index 0000000..38e8971
--- /dev/null
+++ b/tez-dag-api/src/main/proto/DAGPlan.proto
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tez.dag.api";
+option java_outer_classname = "DAGPlan";
+option java_generate_equals_and_hash = true;
+package org.apache.tez.dag.api;
+
+// Many of these types have a dual in the Tez-api. To reduce confusion, these types have prefix or suffix
+// of "Plan" to indicate they are to be used in the dag-plan.
+// The big types use a suffix: JobPlan, VertexPlan, EdgePlan
+// --> these get more direct use in the runtime and the naming is natural.
+// The enums and utility types use prefix: PlanVertexType, PlanEdgeConnectionPaatern, etc
+// --> there is not great naming choice for these that avoids ambiguity, but this one seems acceptable.
+
+enum PlanVertexType {
+ INPUT = 0;
+ NORMAL = 1;
+ OUTPUT = 2;
+}
+
+enum PlanEdgeConnectionPattern {
+ ONE_TO_ONE = 0;
+ ONE_TO_ALL = 1;
+ BIPARTITE = 2;
+}
+
+enum PlanEdgeSourceType {
+ STABLE = 0;
+ STABLE_PERSISTED = 1;
+ STREAMING = 2;
+}
+
+message PlanKeyValuePair {
+ required string key = 1;
+ required string value = 2;
+}
+
+enum PlanLocalResourceType {
+ FILE = 0;
+ ARCHIVE = 1;
+ PATTERN = 2;
+}
+
+enum PlanLocalResourceVisibility {
+ PUBLIC = 0;
+ PRIVATE = 1;
+ APPLICATION = 2;
+}
+
+message PlanLocalResource {
+ required string name = 1;
+ required string uri = 2;
+ required int64 size = 3;
+ required int64 timeStamp = 4;
+ required PlanLocalResourceType type = 5;
+ required PlanLocalResourceVisibility visibility = 6;
+ optional string pattern = 7; // only used if type=PATTERN
+}
+
+// Each taskLocationHint represents a single split in in the input.
+// It is the list of [{rack,machines}] that host a replica of each particular split.
+// For now it is represented as pair-of-arrays rather than array-of-pairs.
+message PlanTaskLocationHint {
+ repeated string rack = 1;
+ repeated string host = 2;
+}
+
+message PlanTaskConfiguration {
+ required int32 numTasks = 1;
+ required int32 memoryMb = 2;
+ required int32 virtualCores = 3;
+ required string javaOpts = 4;
+ required string taskModule = 5;
+ repeated PlanLocalResource localResource = 6;
+ repeated PlanKeyValuePair environmentSetting = 8;
+}
+
+message VertexPlan {
+ required string name = 1;
+ required PlanVertexType type = 2;
+ optional string processorName = 3;
+ required PlanTaskConfiguration taskConfig = 4;
+ repeated PlanTaskLocationHint taskLocationHint = 7;
+ repeated string inEdgeId = 8;
+ repeated string outEdgeId = 9;
+}
+
+message EdgePlan {
+ required string id = 1;
+ required string inputVertexName = 2;
+ required string outputVertexName = 3;
+ required PlanEdgeConnectionPattern connectionPattern = 4;
+ required PlanEdgeSourceType sourceType = 5;
+ required string inputClass = 6;
+ required string outputClass = 7;
+}
+
+message JobPlan {
+ required string name = 1;
+ repeated VertexPlan vertex = 3;
+ repeated EdgePlan edge = 4;
+ repeated PlanKeyValuePair jobSetting = 5;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
new file mode 100644
index 0000000..f89a73e
--- /dev/null
+++ b/tez-dag-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
+import org.apache.tez.dag.api.DAGPlan.*;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+// based on TestDAGLocationHint
+public class TestDAGPlan {
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(); //TODO: doesn't seem to be deleting this folder automatically as expected.
+
+ @Test
+ public void testBasicJobPlanSerde() throws IOException {
+
+ JobPlan job = JobPlan.newBuilder()
+ .setName("test")
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex1")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("machineName").addRack("rack1").build())
+ .setTaskConfig(
+ DAGPlan.PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x.y")
+ .build())
+ .build())
+ .build();
+ File file = tempFolder.newFile("jobPlan");
+ FileOutputStream outStream = null;
+ try {
+ outStream = new FileOutputStream(file);
+ job.writeTo(outStream);
+ }
+ finally {
+ if(outStream != null){
+ outStream.close();
+ }
+ }
+
+ JobPlan inJob;
+ FileInputStream inputStream;
+ try {
+ inputStream = new FileInputStream(file);
+ inJob = JobPlan.newBuilder().mergeFrom(inputStream).build();
+ }
+ finally {
+ outStream.close();
+ }
+
+ Assert.assertEquals(job, inJob);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 37587e4..57003c1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -19,6 +19,7 @@
package org.apache.tez.dag.app;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.EnumSet;
@@ -62,8 +63,9 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.dag.api.DAGConfiguration;
-import org.apache.tez.dag.api.DAGLocationHint;
+import org.apache.tez.dag.api.DAGPlan.JobPlan;
+import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.app.client.ClientService;
import org.apache.tez.dag.app.client.impl.TezClientService;
@@ -131,7 +133,7 @@ public class DAGAppMaster extends CompositeService {
public static final int SHUTDOWN_HOOK_PRIORITY = 30;
private Clock clock;
- private final DAGConfiguration dagPlan;
+ private final JobPlan jobPlan;
private long dagsStartTime;
private final long startTime;
private final long appSubmitTime;
@@ -170,8 +172,6 @@ public class DAGAppMaster extends CompositeService {
private TaskSchedulerEventHandler taskSchedulerEventHandler;
private HistoryEventHandler historyEventHandler;
- private DAGLocationHint dagLocationHint;
-
private DAGAppMasterState state;
private DAG dag;
@@ -180,16 +180,16 @@ public class DAGAppMaster extends CompositeService {
public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
- long appSubmitTime, DAGConfiguration dagPlan) {
+ long appSubmitTime, JobPlan dagPB) {
this(applicationAttemptId, containerId, nmHost, nmPort, nmHttpPort,
- new SystemClock(), appSubmitTime, dagPlan);
+ new SystemClock(), appSubmitTime, dagPB);
}
public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
- Clock clock, long appSubmitTime, DAGConfiguration dagPlan) {
+ Clock clock, long appSubmitTime, JobPlan dagPB) {
super(DAGAppMaster.class.getName());
- this.dagPlan = dagPlan;
+ this.jobPlan = dagPB;
this.clock = clock;
this.startTime = clock.getTime();
this.appSubmitTime = appSubmitTime;
@@ -215,13 +215,12 @@ public class DAGAppMaster extends CompositeService {
conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
downloadTokensAndSetupUGI(conf);
- setupDAGLocationHint(dagPlan);
context = new RunningAppContext(conf);
// Job name is the same as the app name util we support DAG of jobs
// for an app later
- appName = dagPlan.getName();
+ appName = jobPlan.getName();
dagId = new TezDAGID(appAttemptID.getApplicationId(), 1);
@@ -521,11 +520,11 @@ public class DAGAppMaster extends CompositeService {
// }
/** Create and initialize (but don't start) a single dag. */
- protected DAG createDAG(DAGConfiguration dagPlan) {
+ protected DAG createDAG(JobPlan dagPB) {
// create single job
DAG newDag =
- new DAGImpl(dagId, appAttemptID, conf, dagPlan, dispatcher.getEventHandler(),
+ new DAGImpl(dagId, appAttemptID, conf, dagPB, dispatcher.getEventHandler(),
taskAttemptListener, jobTokenSecretManager, fsTokens, clock,
// TODO Recovery
//completedTasksFromPreviousRun,
@@ -534,7 +533,7 @@ public class DAGAppMaster extends CompositeService {
//committer, newApiCommitter,
currentUser.getShortUserName(), appSubmitTime,
//amInfos,
- taskHeartbeatHandler, context, dagLocationHint);
+ taskHeartbeatHandler, context);
((RunningAppContext) context).setDAG(newDag);
dispatcher.register(DAGFinishEvent.Type.class,
@@ -578,23 +577,6 @@ public class DAGAppMaster extends CompositeService {
}
}
- protected void setupDAGLocationHint(DAGConfiguration conf) {
- try {
- String dagLocationHintFile =
- conf.get(DAGConfiguration.DAG_LOCATION_HINT_RESOURCE_FILE,
- DAGConfiguration.DEFAULT_DAG_LOCATION_HINT_RESOURCE_FILE);
- File f = new File(dagLocationHintFile);
- if (f.exists()) {
- this.dagLocationHint = DAGLocationHint.initDAGDagLocationHint(
- dagLocationHintFile);
- } else {
- this.dagLocationHint = new DAGLocationHint();
- }
- } catch (IOException e) {
- throw new YarnException(e);
- }
- }
-
protected void addIfService(Object object) {
if (object instanceof Service) {
addService((Service) object);
@@ -846,7 +828,7 @@ public class DAGAppMaster extends CompositeService {
*/
// /////////////////// Create the job itself.
- dag = createDAG(dagPlan);
+ dag = createDAG(jobPlan);
// End of creating the job.
@@ -973,27 +955,41 @@ public class DAGAppMaster extends CompositeService {
// Default to running mr if nothing specified.
// TODO change this once the client is ready.
String type;
- DAGConfiguration dagPlan = null;
TezConfiguration conf = new TezConfiguration(new YarnConfiguration());
+
+ JobPlan jobPlan = null;
if (cliParser.hasOption(OPT_PREDEFINED)) {
LOG.info("Running with PreDefined configuration");
type = cliParser.getOptionValue(OPT_PREDEFINED, "mr");
LOG.info("Running job type: " + type);
if (type.equals("mr")) {
- dagPlan = (DAGConfiguration)MRRExampleHelper.createDAGConfigurationForMR();
+ jobPlan = MRRExampleHelper.createDAGConfigurationForMR();
} else if (type.equals("mrr")) {
- dagPlan = (DAGConfiguration)MRRExampleHelper.createDAGConfigurationForMRR();
+ jobPlan = MRRExampleHelper.createDAGConfigurationForMRR();
+ }
+ }
+ else {
+ // Read the protobuf DAG
+ JobPlan.Builder dagPlanBuilder = JobPlan.newBuilder();
+ FileInputStream dagPBBinaryStream = null;
+ try {
+ dagPBBinaryStream = new FileInputStream(TezConfiguration.DAG_AM_PLAN_PB_BINARY);
+ dagPlanBuilder.mergeFrom(dagPBBinaryStream);
}
- } else {
- dagPlan = new DAGConfiguration();
- dagPlan.addResource(TezConfiguration.DAG_AM_PLAN_CONFIG_XML);
+ finally {
+ if(dagPBBinaryStream != null){
+ dagPBBinaryStream.close();
+ }
+ }
+
+ jobPlan = dagPlanBuilder.build();
}
LOG.info("XXXX Running a DAG with "
- + dagPlan.getVertices().length + " vertices ");
- for (String v : dagPlan.getVertices()) {
- LOG.info("XXXX DAG has vertex " + v);
+ + jobPlan.getVertexCount() + " vertices ");
+ for (VertexPlan v : jobPlan.getVertexList()) {
+ LOG.info("XXXX DAG has vertex " + v.getName());
}
String jobUserName = System
@@ -1004,7 +1000,7 @@ public class DAGAppMaster extends CompositeService {
// the objects myself.
conf.setBoolean("fs.automatic.close", false);
- Map<String, String> config = dagPlan.getConfig();
+ Map<String, String> config = DagTypeConverters.createSettingsMapFromDAGPlan(jobPlan.getJobSettingList());
for(Entry<String, String> entry : config.entrySet()) {
conf.set(entry.getKey(), entry.getValue());
}
@@ -1012,7 +1008,7 @@ public class DAGAppMaster extends CompositeService {
DAGAppMaster appMaster =
new DAGAppMaster(applicationAttemptId, containerId, nodeHostString,
Integer.parseInt(nodePortString),
- Integer.parseInt(nodeHttpPortString), appSubmitTime, dagPlan);
+ Integer.parseInt(nodeHttpPortString), appSubmitTime, jobPlan);
ShutdownHookManager.get().addShutdownHook(
new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
index 1b69919..b50887a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
@@ -17,7 +17,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.tez.dag.api.DAGConfiguration;
+import org.apache.tez.dag.api.DAGPlan.JobPlan;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
@@ -84,7 +84,10 @@ public class MRRExampleHelper {
return resourceNames;
}
- static Configuration createDAGConfigurationForMRR() throws IOException {
+ // TODO: these preconfigured jobs seem to require User and perhaps some other work.
+ // -> not tested with new DagPB system.
+
+ static JobPlan createDAGConfigurationForMRR() throws IOException {
org.apache.tez.dag.api.DAG dag = new org.apache.tez.dag.api.DAG();
Vertex mapVertex = new Vertex("map",
"org.apache.tez.mapreduce.task.InitialTask", 6);
@@ -143,16 +146,15 @@ public class MRRExampleHelper {
dag.addEdge(edge1);
dag.addEdge(edge2);
dag.verify();
- DAGConfiguration dagConf = dag.serializeDag();
-
- dagConf.setBoolean(MRJobConfig.MAP_SPECULATIVE, false);
- dagConf.setBoolean(MRJobConfig.REDUCE_SPECULATIVE, false);
-
- return dagConf;
+ dag.addConfiguration(MRJobConfig.MAP_SPECULATIVE, new Boolean(false).toString());
+ dag.addConfiguration(MRJobConfig.REDUCE_SPECULATIVE, new Boolean(false).toString());
+
+ JobPlan dagPB = dag.createDag();
+ return dagPB;
}
// TODO remove once client is in place
- static Configuration createDAGConfigurationForMR() throws IOException {
+ static JobPlan createDAGConfigurationForMR() throws IOException {
org.apache.tez.dag.api.DAG dag = new org.apache.tez.dag.api.DAG();
Vertex mapVertex = new Vertex("map",
"org.apache.tez.mapreduce.task.InitialTask", 6);
@@ -194,12 +196,13 @@ public class MRRExampleHelper {
dag.addVertex(reduceVertex);
dag.addEdge(edge);
dag.verify();
- DAGConfiguration dagConf = dag.serializeDag();
-
- dagConf.setBoolean(MRJobConfig.MAP_SPECULATIVE, false);
- dagConf.setBoolean(MRJobConfig.REDUCE_SPECULATIVE, false);
-
- return dagConf;
+
+ dag.addConfiguration(MRJobConfig.MAP_SPECULATIVE, new Boolean(false).toString());
+ dag.addConfiguration(MRJobConfig.REDUCE_SPECULATIVE, new Boolean(false).toString());
+
+ JobPlan dagPB = dag.createDag();
+
+ return dagPB;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index 03bf570..39cfc1a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -24,7 +24,7 @@ import java.util.Map;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGConfiguration;
+import org.apache.tez.dag.api.DAGPlan.JobPlan;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.engine.records.TezDAGID;
@@ -60,8 +60,8 @@ public interface DAG {
TezConfiguration getConf();
- DAGConfiguration getDagPlan();
-
+ JobPlan getJobPlan();
+
/**
* @return the ACLs for this job for each type of JobACL given.
*/
@@ -75,5 +75,4 @@ public interface DAG {
boolean checkAccess(UserGroupInformation callerUGI, ApplicationAccessType jobOperation);
- VertexLocationHint getVertexLocationHint(TezVertexID vertexId);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index 5d8b8f7..8d9fbc3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -28,6 +28,7 @@ import org.apache.tez.common.InputSpec;
import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.DAGPlan.VertexPlan;
import org.apache.tez.engine.records.TezDependentTaskCompletionEvent;
import org.apache.tez.engine.records.TezTaskID;
import org.apache.tez.engine.records.TezVertexID;
@@ -39,6 +40,8 @@ import org.apache.tez.engine.records.TezVertexID;
public interface Vertex extends Comparable<Vertex> {
TezVertexID getVertexId();
+ public VertexPlan getVertexPlan();
+
int getDistanceFromRoot();
String getName();
VertexState getState();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 0e1206f..2e20692 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
@@ -53,13 +52,15 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGConfiguration;
-import org.apache.tez.dag.api.DAGLocationHint;
+import org.apache.tez.dag.api.DAGPlan.EdgePlan;
+import org.apache.tez.dag.api.DAGPlan.JobPlan;
+import org.apache.tez.dag.api.DAGPlan.VertexPlan;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.impl.TezBuilderUtils;
+import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
@@ -124,16 +125,14 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private final long appSubmitTime;
private final AppContext appContext;
- volatile Map<TezVertexID, Vertex> vertices =
- new HashMap<TezVertexID, Vertex>();
- private Map<String, EdgeProperty> edgeProperties =
- new HashMap<String, EdgeProperty>();
+ volatile Map<TezVertexID, Vertex> vertices = new HashMap<TezVertexID, Vertex>();
+ private Map<String, EdgeProperty> edges = new HashMap<String, EdgeProperty>();
private TezCounters dagCounters = new TezCounters();
private Object fullCountersLock = new Object();
private TezCounters fullCounters = null;
public final TezConfiguration conf;
- public final DAGConfiguration dagPlan;
+ private final JobPlan jobPlan;
//fields initialized in init
private FileSystem fs;
@@ -151,9 +150,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private static final DAGSchedulerUpdateTransition
DAG_SCHEDULER_UPDATE_TRANSITION = new DAGSchedulerUpdateTransition();
- // Location hints for all vertices in DAG
- private final DAGLocationHint dagLocationHint;
-
protected static final
StateMachineFactory<DAGImpl, DAGState, DAGEventType, DAGEvent>
stateMachineFactory
@@ -315,7 +311,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
public DAGImpl(TezDAGID dagId, ApplicationAttemptId applicationAttemptId,
TezConfiguration conf,
- DAGConfiguration dagPlan,
+ JobPlan jobPlan,
EventHandler eventHandler,
TaskAttemptListener taskAttemptListener,
JobTokenSecretManager jobTokenSecretManager,
@@ -327,14 +323,13 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
// TODO Recovery
//List<AMInfo> amInfos,
TaskHeartbeatHandler thh,
- AppContext appContext,
- DAGLocationHint dagLocationHint) {
+ AppContext appContext) {
this.applicationAttemptId = applicationAttemptId;
this.dagId = dagId;
- this.dagPlan = dagPlan;
+ this.jobPlan = jobPlan;
this.conf = conf;
- this.dagName = (dagPlan.getName() != null) ? dagPlan.getName() :
- "<missing app name>";
+ this.dagName = (jobPlan.getName() != null) ? jobPlan.getName() : "<missing app name>";
+
this.userName = appUserName;
// TODO Metrics
//this.metrics = metrics;
@@ -357,8 +352,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
this.aclsManager = new ApplicationACLsManager(conf);
- this.dagLocationHint = dagLocationHint;
-
// This "this leak" is okay because the retained pointer is in an
// instance variable.
stateMachine = stateMachineFactory.make(this);
@@ -380,8 +373,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
@Override
- public DAGConfiguration getDagPlan() {
- return dagPlan;
+ public JobPlan getJobPlan() {
+ return jobPlan;
}
EventHandler getEventHandler() {
@@ -812,18 +805,19 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
*/
// create the vertices
- String[] vertexNames = dag.getDagPlan().getVertices();
- dag.numVertices = vertexNames.length;
+ dag.numVertices = dag.getJobPlan().getVertexCount();
for (int i=0; i < dag.numVertices; ++i) {
- VertexImpl v = createVertex(dag, vertexNames[i], i);
+ String vertexName = dag.getJobPlan().getVertex(i).getName();
+ VertexImpl v = createVertex(dag, vertexName, i);
dag.addVertex(v);
}
- dag.edgeProperties = dag.getDagPlan().getEdgeProperties();
-
+ dag.edges = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dag.getJobPlan().getEdgeList());
+ Map<String,EdgePlan> edgePlans = DagTypeConverters.createEdgePlanMapFromDAGPlan(dag.getJobPlan().getEdgeList());
+
// setup the dag
for (Vertex v : dag.vertices.values()) {
- parseVertexEdges(dag, v);
+ parseVertexEdges(dag, edgePlans, v);
}
dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
@@ -846,39 +840,43 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
TezVertexID vertexId = TezBuilderUtils.newVertexID(dag.getID(), vId);
+
+ VertexPlan vertexPlan = dag.getJobPlan().getVertex(vId);
+ VertexLocationHint vertexLocationHint = DagTypeConverters.convertFromDAGPlan(vertexPlan.getTaskLocationHintList());
+
return new VertexImpl(
- vertexId, vertexName, dag.conf,
+ vertexId, vertexPlan, vertexName, dag.conf,
dag.eventHandler, dag.taskAttemptListener,
dag.jobToken, dag.fsTokens, dag.clock,
dag.taskHeartbeatHandler, dag.appContext,
- dag.dagLocationHint.getVertexLocationHint(vertexName));
+ vertexLocationHint);
}
- private void parseVertexEdges(DAGImpl dag, Vertex vertex) {
- String[] inVerticesNames =
- dag.getDagPlan().getInputVertices(vertex.getName());
- List<String> inEdges =
- dag.getDagPlan().getInputEdgeIds(vertex.getName());
+ // hooks up this VertexImpl to input and output EdgeProperties
+ private void parseVertexEdges(DAGImpl dag, Map<String, EdgePlan> edgePlans, Vertex vertex) {
+ VertexPlan vertexPlan = vertex.getVertexPlan();
+
Map<Vertex, EdgeProperty> inVertices =
new HashMap<Vertex, EdgeProperty>();
- for (int i=0; i < inVerticesNames.length; ++i) {
- String vertexName = inVerticesNames[i];
- inVertices.put(dag.getVertex(vertexName),
- dag.edgeProperties.get(inEdges.get(i)));
- }
- vertex.setInputVertices(inVertices);
- String[] outVerticesNames =
- dag.getDagPlan().getOutputVertices(vertex.getName());
- List<String> outEdges =
- dag.getDagPlan().getOutputEdgeIds(vertex.getName());
Map<Vertex, EdgeProperty> outVertices =
new HashMap<Vertex, EdgeProperty>();
- for (int i=0; i < outVerticesNames.length; ++i) {
- String vertexName = outVerticesNames[i];
- outVertices.put(dag.getVertex(vertexName),
- dag.edgeProperties.get(outEdges.get(i)));
+
+ for(String inEdgeId : vertexPlan.getInEdgeIdList()){
+ EdgePlan edgePlan = edgePlans.get(inEdgeId);
+ Vertex inVertex = dag.vertexMap.get(edgePlan.getInputVertexName());
+ EdgeProperty edgeProp = dag.edges.get(inEdgeId);
+ inVertices.put(inVertex, edgeProp);
}
+
+ for(String outEdgeId : vertexPlan.getOutEdgeIdList()){
+ EdgePlan edgePlan = edgePlans.get(outEdgeId);
+ Vertex outVertex = dag.vertexMap.get(edgePlan.getOutputVertexName());
+ EdgeProperty edgeProp = dag.edges.get(outEdgeId);
+ outVertices.put(outVertex, edgeProp);
+ }
+
+ vertex.setInputVertices(inVertices);
vertex.setOutputVertices(outVertices);
}
@@ -1145,11 +1143,4 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
job.finished(DAGState.ERROR);
}
}
-
- @Override
- public VertexLocationHint getVertexLocationHint(TezVertexID vertexId) {
- return dagLocationHint.getVertexLocationHint(
- getVertex(vertexId).getName());
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 80de958..a423db1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -55,7 +55,8 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.tez.common.InputSpec;
import org.apache.tez.common.OutputSpec;
import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.DAGConfiguration;
+import org.apache.tez.dag.api.DAGPlan.VertexPlan;
+import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
import org.apache.tez.dag.api.EdgeProperty;
@@ -324,7 +325,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private Credentials fsTokens;
private Token<JobTokenIdentifier> jobToken;
- private final TezVertexID vertexId;
+ private final TezVertexID vertexId; //runtime assigned id.
+ private final VertexPlan vertexPlan;
private final String vertexName;
private final String processorName;
@@ -341,7 +343,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private Map<String, String> environment;
private String javaOpts;
- public VertexImpl(TezVertexID vertexId, String vertexName,
+ public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, String vertexName,
TezConfiguration conf, EventHandler eventHandler,
TaskAttemptListener taskAttemptListener,
Token<JobTokenIdentifier> jobToken,
@@ -353,6 +355,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
TaskHeartbeatHandler thh,
AppContext appContext, VertexLocationHint vertexLocationHint) {
this.vertexId = vertexId;
+ this.vertexPlan = vertexPlan;
this.vertexName = vertexName;
this.conf = conf;
//this.metrics = metrics;
@@ -373,12 +376,11 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
this.committer = new NullVertexOutputCommitter();
this.vertexLocationHint = vertexLocationHint;
- this.taskResource = getDAGPlan().getVertexResource(getName());
- this.processorName = getDAGPlan().getVertexTaskModuleClassName(getName());
- this.localResources = getDAGPlan().getVertexLocalResources(getName());
- this.environment = getDAGPlan().getVertexEnv(getName());
-
- this.javaOpts = getDAGPlan().getVertexJavaOpts(getName());
+ this.taskResource = DagTypeConverters.CreateResourceRequestFromTaskConfig(vertexPlan.getTaskConfig());
+ this.processorName = vertexPlan.hasProcessorName() ? vertexPlan.getProcessorName() : null;
+ this.localResources = DagTypeConverters.createLocalResourceMapFromDAGPlan(vertexPlan.getTaskConfig().getLocalResourceList());
+ this.environment = DagTypeConverters.createEnvironmentMapFromDAGPlan(vertexPlan.getTaskConfig().getEnvironmentSettingList());
+ this.javaOpts = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan.getTaskConfig().getJavaOpts() : null;
// This "this leak" is okay because the retained pointer is in an
// instance variable.
@@ -395,6 +397,11 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
@Override
+ public VertexPlan getVertexPlan() {
+ return vertexPlan;
+ }
+
+ @Override
public int getDistanceFromRoot() {
return distanceFromRoot;
}
@@ -727,7 +734,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// TODODAGAM
// TODO: Splits?
- vertex.numTasks = vertex.getDAGPlan().getNumVertexTasks(vertex.getName());
+
+ vertex.numTasks = vertex.getVertexPlan().getTaskConfig().getNumTasks();
+
/*
TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(job, job.jobId);
job.numMapTasks = taskSplitMetaInfo.length;
@@ -1217,10 +1226,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
return appContext.getDAG();
}
- DAGConfiguration getDAGPlan() {
- return getDAG().getDagPlan();
- }
-
// TODO Eventually remove synchronization.
@Override
public synchronized List<InputSpec> getInputSpecList() {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
index 331e2a3..b3db770 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezEngineChildJVM.java
@@ -82,6 +82,12 @@ public class TezEngineChildJVM {
//set custom javaOpts
vargs.add(javaOpts);
+//[Debug Task] Current simplest way to attach debugger to Tez Child Task
+// Uncomment the following, then launch a regular job
+// Works best on one-box configured with a single container (hence one task at a time).
+// LOG.error(" !!!!!!!!! Launching Child-Task in debug/suspend mode. Attach to port 8003 !!!!!!!!");
+// vargs.add("-Xdebug -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=8003,server=y,suspend=y");
+
Path childTmpDir = new Path(Environment.PWD.$(),
YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
vargs.add("-Djava.io.tmpdir=" + childTmpDir);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cb5758b4/tez-dag/src/test/java/org/apache/tez/dag/api/TestDAGLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/api/TestDAGLocationHint.java b/tez-dag/src/test/java/org/apache/tez/dag/api/TestDAGLocationHint.java
deleted file mode 100644
index 3deaee7..0000000
--- a/tez-dag/src/test/java/org/apache/tez/dag/api/TestDAGLocationHint.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestDAGLocationHint {
-
- private DataInput in;
- private DataOutput out;
- private ByteArrayOutputStream bOut;
-
- @Before
- public void setup() {
- bOut = new ByteArrayOutputStream();
- out = new DataOutputStream(bOut);
- }
-
- @After
- public void teardown() {
- in = null;
- out = null;
- bOut = null;
- }
-
- @Test
- public void testNullDAGLocationHintSerDes() throws IOException {
- DAGLocationHint expected = new DAGLocationHint();
- expected.write(out);
- in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
- DAGLocationHint actual = new DAGLocationHint();
- actual.readFields(in);
- Assert.assertNotNull(actual.getVertexLocationHints());
- Assert.assertEquals(0, actual.getVertexLocationHints().size());
- }
-
- @Test
- public void testDAGLocationHintSerDes() throws IOException {
- String[] hosts = { "h1", "h2", "", null };
- String[] racks = { "r1", "r2" };
-
- VertexLocationHint vertexLocationHint = new VertexLocationHint(4);
- vertexLocationHint.getTaskLocationHints()[0] =
- new TaskLocationHint(hosts, racks);
- DAGLocationHint expected = new DAGLocationHint();
- expected.getVertexLocationHints().put("v1", null);
- expected.getVertexLocationHints().put("v2", new VertexLocationHint());
- expected.getVertexLocationHints().put("v3", vertexLocationHint);
- expected.write(out);
-
- in = new DataInputStream(new ByteArrayInputStream(bOut.toByteArray()));
- DAGLocationHint actual = new DAGLocationHint();
- actual.readFields(in);
- Assert.assertNotNull(actual.getVertexLocationHints());
- Assert.assertEquals(3, actual.getVertexLocationHints().size());
-
- Assert.assertNull(actual.getVertexLocationHint("v1"));
- Assert.assertNotNull(actual.getVertexLocationHint("v2"));
- Assert.assertNotNull(actual.getVertexLocationHint("v3"));
-
- Assert.assertEquals(0, actual.getVertexLocationHint("v2").getNumTasks());
- Assert.assertEquals(0,
- actual.getVertexLocationHint("v2").getTaskLocationHints().length);
-
- Assert.assertEquals(4, actual.getVertexLocationHint("v3").getNumTasks());
- Assert.assertEquals(4,
- actual.getVertexLocationHint("v3").getTaskLocationHints().length);
- Assert.assertNotNull(
- actual.getVertexLocationHint("v3").getTaskLocationHints()[0]);
- Assert.assertArrayEquals(racks,
- actual.getVertexLocationHint("v3").getTaskLocationHints()[0].
- getRacks());
- Assert.assertArrayEquals(hosts,
- actual.getVertexLocationHint("v3").getTaskLocationHints()[0].
- getDataLocalHosts());
- Assert.assertNull(
- actual.getVertexLocationHint("v3").getTaskLocationHints()[1]);
- Assert.assertNull(
- actual.getVertexLocationHint("v3").getTaskLocationHints()[2]);
- Assert.assertNull(
- actual.getVertexLocationHint("v3").getTaskLocationHints()[3]);
- }
-
-
-}
-
[31/50] [abbrv] git commit: TEZ-137 VertexHistoryEvents are using
VertexStatus.state instead of VertexImpl.state (bikas)
Posted by ss...@apache.org.
TEZ-137 VertexHistoryEvents are using VertexStatus.state instead of VertexImpl.state (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/3bb954bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/3bb954bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/3bb954bd
Branch: refs/heads/master
Commit: 3bb954bd9153f1b6b7e394776d9169cd99acde1b
Parents: a2a7261
Author: Bikas Saha <bi...@apache.org>
Authored: Tue May 28 13:33:52 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue May 28 13:33:52 2013 -0700
----------------------------------------------------------------------
.../main/java/org/apache/tez/client/TezClient.java | 3 -
.../hadoop/mapred/MRVertexOutputCommitter.java | 19 +++++-
.../apache/tez/dag/api/committer/DAGStatus.java | 39 -----------
.../api/committer/NullVertexOutputCommitter.java | 4 +-
.../dag/api/committer/VertexOutputCommitter.java | 1 +
.../apache/tez/dag/api/committer/VertexStatus.java | 51 ---------------
.../org/apache/tez/dag/app/dag/impl/DAGImpl.java | 2 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 12 ++--
.../tez/dag/history/events/DAGFinishedEvent.java | 2 +-
.../dag/history/events/VertexFinishedEvent.java | 2 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 4 +-
11 files changed, 32 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
index f794014..73c97b3 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezRemoteException;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.VertexStatus;
@@ -45,7 +44,6 @@ public class TezClient {
public DAGClient getDAGClient(String appIdStr) throws IOException, TezException {
try {
- System.out.println("Fetching app: " + appIdStr);
ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
YarnClient yarnClient = new YarnClientImpl();
yarnClient.init(conf);
@@ -60,7 +58,6 @@ public class TezClient {
}
public DAGClient getDAGClient(String host, int port) throws IOException {
- System.out.println("App host port: " + host + ":" + port);
InetSocketAddress addr = new InetSocketAddress(host, port);
DAGClient dagClient;
dagClient = new DAGClientRPCImpl(1, addr, conf);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
index 7c886d6..171c244 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
@@ -33,9 +33,10 @@ import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.YarnException;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.client.VertexStatus;
import org.apache.tez.dag.api.committer.VertexContext;
import org.apache.tez.dag.api.committer.VertexOutputCommitter;
-import org.apache.tez.dag.api.committer.VertexStatus;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.utils.TezBuilderUtils;
@@ -102,7 +103,21 @@ public class MRVertexOutputCommitter extends VertexOutputCommitter {
}
private State getJobStateFromVertexStatusState(VertexStatus.State state) {
- return JobStatus.State.valueOf(state.name());
+ switch(state) {
+ case INITED:
+ return JobStatus.State.PREP;
+ case RUNNING:
+ return JobStatus.State.RUNNING;
+ case SUCCEEDED:
+ return JobStatus.State.SUCCEEDED;
+ case KILLED:
+ return JobStatus.State.KILLED;
+ case FAILED:
+ case ERROR:
+ return JobStatus.State.FAILED;
+ default:
+ throw new TezException("Unknown VertexStatus.State: " + state);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
deleted file mode 100644
index 587e18f..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/DAGStatus.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.committer;
-
-public class DAGStatus {
- public static enum State {
- RUNNING(1),
- SUCCEEDED(2),
- FAILED(3),
- PREP(4),
- KILLED(5);
-
- int value;
-
- State(int value) {
- this.value = value;
- }
-
- public int getValue() {
- return value;
- }
-
- };
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
index c284ab5..08b2d4a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
@@ -20,7 +20,7 @@ package org.apache.tez.dag.api.committer;
import java.io.IOException;
-import org.apache.tez.dag.api.committer.VertexStatus.State;
+import org.apache.tez.dag.api.client.VertexStatus;
public class NullVertexOutputCommitter extends VertexOutputCommitter {
@@ -40,7 +40,7 @@ public class NullVertexOutputCommitter extends VertexOutputCommitter {
}
@Override
- public void abortVertex(State finalState) throws IOException {
+ public void abortVertex(VertexStatus.State finalState) throws IOException {
// Nothing to do
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
index b814d0e..34c6e89 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.tez.dag.api.client.VertexStatus;
@Public
@Unstable
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
deleted file mode 100644
index a78027a..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexStatus.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.tez.dag.api.committer;
-
-public class VertexStatus {
-
- public static enum State {
- RUNNING(1),
- SUCCEEDED(2),
- FAILED(3),
- PREP(4), // TODO change to INITING
- KILLED(5);
-
- int value;
-
- State(int value) {
- this.value = value;
- }
-
- public int getValue() {
- return value;
- }
-
- };
-
- private State runState;
-
- public synchronized State getState() {
- return runState;
- }
-
- public synchronized void setState(State state) {
- this.runState = state;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 4c63d8f..cc6699d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -59,10 +59,10 @@ import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.VertexLocationHint;
+import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.DAGStatusBuilder;
import org.apache.tez.dag.api.client.ProgressBuilder;
import org.apache.tez.dag.api.client.VertexStatusBuilder;
-import org.apache.tez.dag.api.committer.DAGStatus;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index ce4c609..e34382f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -61,15 +61,15 @@ import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
import org.apache.tez.dag.api.client.ProgressBuilder;
+import org.apache.tez.dag.api.client.VertexStatus;
import org.apache.tez.dag.api.client.VertexStatusBuilder;
import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
import org.apache.tez.dag.api.committer.VertexContext;
import org.apache.tez.dag.api.committer.VertexOutputCommitter;
-import org.apache.tez.dag.api.committer.VertexStatus;
-import org.apache.tez.dag.api.committer.VertexStatus.State;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
@@ -744,23 +744,25 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
case KILL_WAIT:
eventHandler.handle(new DAGEventVertexCompleted(getVertexId(),
finalState));
- logJobHistoryVertexFailedEvent(State.KILLED);
+ logJobHistoryVertexFailedEvent(VertexStatus.State.KILLED);
break;
case ERROR:
eventHandler.handle(new DAGEvent(getDAGId(),
DAGEventType.INTERNAL_ERROR));
- logJobHistoryVertexFailedEvent(State.FAILED);
+ logJobHistoryVertexFailedEvent(VertexStatus.State.FAILED);
break;
case FAILED:
eventHandler.handle(new DAGEventVertexCompleted(getVertexId(),
finalState));
- logJobHistoryVertexFailedEvent(State.FAILED);
+ logJobHistoryVertexFailedEvent(VertexStatus.State.FAILED);
break;
case SUCCEEDED:
eventHandler.handle(new DAGEventVertexCompleted(getVertexId(),
finalState));
logJobHistoryVertexFinishedEvent();
break;
+ default:
+ throw new TezException("Unexpected VertexState: " + finalState);
}
return finalState;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
index 5dea9a7..8794cea 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.history.events;
-import org.apache.tez.dag.api.committer.DAGStatus;
+import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.DAGFinished;
import org.apache.tez.dag.history.avro.HistoryEventType;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
index 91c209f..7e86c66 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
@@ -18,7 +18,7 @@
package org.apache.tez.dag.history.events;
-import org.apache.tez.dag.api.committer.VertexStatus;
+import org.apache.tez.dag.api.client.VertexStatus;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.avro.HistoryEventType;
import org.apache.tez.dag.history.avro.VertexFinished;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3bb954bd/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index f78a7b8..e4a1040 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -50,10 +50,10 @@ import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint;
+import org.apache.tez.dag.api.client.VertexStatus;
import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
import org.apache.tez.dag.api.committer.VertexContext;
import org.apache.tez.dag.api.committer.VertexOutputCommitter;
-import org.apache.tez.dag.api.committer.VertexStatus.State;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
@@ -147,7 +147,7 @@ public class TestVertexImpl {
}
@Override
- public void abortVertex(State finalState) throws IOException {
+ public void abortVertex(VertexStatus.State finalState) throws IOException {
++abortCounter;
if (throwErrorOnAbort) {
throw new IOException("I can throwz exceptions in abort");